From 5ee09790f05cc70cd0bf274d996435ba03a56554 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Fri, 16 Aug 2024 12:56:38 +0300 Subject: [PATCH 01/86] [#126] Fix docker warnings Signed-off-by: Roman Loginov --- .docker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index d39fba1..f45c864 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,9 +1,9 @@ -FROM golang:1.22-alpine as basebuilder +FROM golang:1.22-alpine AS basebuilder RUN apk add --update make bash ca-certificates -FROM basebuilder as builder -ENV GOGC off -ENV CGO_ENABLED 0 +FROM basebuilder AS builder +ENV GOGC=off +ENV CGO_ENABLED=0 ARG BUILD=now ARG VERSION=dev ARG REPO=repository From 151e5bc1c835a62f1d950647031109b4f02530f8 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 23 Aug 2024 13:19:08 +0300 Subject: [PATCH 02/86] [#132] Update Go version Signed-off-by: Nikita Zinkevich --- .forgejo/workflows/builds.yml | 2 +- .forgejo/workflows/dco.yml | 2 +- .forgejo/workflows/tests.yml | 6 +++--- .golangci.yml | 3 ++- .pre-commit-config.yaml | 5 ----- CHANGELOG.md | 3 +++ Makefile | 4 ++-- go.mod | 2 +- internal/handler/reader_test.go | 2 +- utils/tracing.go | 4 ++-- 10 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.forgejo/workflows/builds.yml b/.forgejo/workflows/builds.yml index 17f1f2e..490a97c 100644 --- a/.forgejo/workflows/builds.yml +++ b/.forgejo/workflows/builds.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.21', '1.22' ] + go_versions: [ '1.22', '1.23' ] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index eb23ec5..4acd633 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.23' - name: Run commit format checker uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 74e0b2c..db7f986 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.23' cache: true - name: Install linters @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.21', '1.22' ] + go_versions: [ '1.22', '1.23' ] fail-fast: false steps: - uses: actions/checkout@v3 @@ -38,4 +38,4 @@ jobs: run: make dep - name: Run tests - run: make test \ No newline at end of file + run: make test diff --git a/.golangci.yml b/.golangci.yml index 5459bde..d9f93eb 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -12,7 +12,8 @@ run: # output configuration options output: # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" - format: tab + formats: + - format: tab # all available settings of specific linters linters-settings: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e97fc23..3c963be 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,11 +30,6 @@ repos: hooks: - id: shellcheck - - repo: https://github.com/golangci/golangci-lint - rev: v1.51.2 - hooks: - - id: golangci-lint - - repo: local hooks: - id: make-lint-install diff --git a/CHANGELOG.md b/CHANGELOG.md index cf47b00..b322c96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Changed +- Update go version to 1.22 (#132) + ## [0.30.0] - Kangshung - 2024-07-22 ### Fixed diff --git a/Makefile b/Makefile index 04cfea4..372b89b 100755 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") GO_VERSION ?= 1.22 -LINT_VERSION ?= 1.54.0 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.2 +LINT_VERSION ?= 1.60.3 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.6 BUILD ?= $(shell date -u --iso=seconds) HUB_IMAGE ?= truecloudlab/frostfs-http-gw diff --git a/go.mod b/go.mod index 493a003..4e4bf9c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw -go 1.21 +go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 diff --git a/internal/handler/reader_test.go b/internal/handler/reader_test.go index 73899ca..c63a734 100644 --- a/internal/handler/reader_test.go +++ b/internal/handler/reader_test.go @@ -35,7 +35,7 @@ func TestDetector(t *testing.T) { } { t.Run(tc.Name, func(t *testing.T) { contentType, data, err := readContentType(uint64(len(tc.Expected)), - func(sz uint64) (io.Reader, error) { + func(uint64) (io.Reader, error) { return strings.NewReader(tc.Expected), nil }, ) diff --git a/utils/tracing.go b/utils/tracing.go index 14c059a..c8e467d 100644 --- a/utils/tracing.go +++ b/utils/tracing.go @@ -30,12 +30,12 @@ func (c *httpCarrier) Set(key string, value string) { func (c *httpCarrier) Keys() []string { dict := make(map[string]interface{}) c.r.Request.Header.VisitAll( - func(key, value []byte) { + func(key, _ []byte) { dict[string(key)] = true }, ) c.r.Response.Header.VisitAll( - func(key, value []byte) { + func(key, _ []byte) { dict[string(key)] = true }, ) From ca426fff4df117b5ce119d7a5f123b11cc99a7a1 Mon Sep 17 00:00:00 2001 From: Roman Ognev Date: Fri, 30 Aug 2024 20:06:01 +0300 Subject: [PATCH 03/86] [#135] Add fuzzing tests for handlers Signed-off-by: Roman Ognev --- Makefile | 36 +- README.md | 23 + go.mod | 1 + go.sum | 2 + internal/handler/handler_fuzz_test.go | 580 ++++++++++++++++++++++++++ 5 files changed, 641 insertions(+), 1 deletion(-) create mode 100644 internal/handler/handler_fuzz_test.go diff --git a/Makefile b/Makefile index 372b89b..c1f4f50 100755 --- a/Makefile +++ b/Makefile @@ -30,6 +30,11 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed "s/-/~/")-${OS_RELEASE} .PHONY: debpackage debclean +FUZZ_NGFUZZ_DIR ?= "" +FUZZ_TIMEOUT ?= 30 +FUZZ_FUNCTIONS ?= "all" +FUZZ_AUX ?= "" + # Make all binaries all: $(BINS) $(BINS): $(DIRS) dep @@ -78,6 +83,35 @@ cover: @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic @go tool cover -html=coverage.txt -o coverage.html +# Run fuzzing +CLANG := $(shell which clang-17 2>/dev/null) +.PHONY: check-clang all +check-clang: +ifeq ($(CLANG),) + @echo "clang-17 is not installed. Please install it before proceeding - https://apt.llvm.org/llvm.sh " + @exit 1 +endif + +.PHONY: check-ngfuzz all +check-ngfuzz: + @if [ -z "$(FUZZ_NGFUZZ_DIR)" ]; then \ + echo "Please set a variable FUZZ_NGFUZZ_DIR to specify path to the ngfuzz"; \ + exit 1; \ + fi + +.PHONY: install-fuzzing-deps +install-fuzzing-deps: check-clang check-ngfuzz + +.PHONY: fuzz +fuzz: install-fuzzing-deps + @START_PATH=$$(pwd); \ + ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \ + cd $(FUZZ_NGFUZZ_DIR) && \ + ./ngfuzz -clean && \ + ./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \ + ./ngfuzz -report + + # Reformat code fmt: @echo "⇒ Processing gofmt check" @@ -149,7 +183,7 @@ version: # Clean up clean: rm -rf vendor - rm -rf $(BINDIR) + rm -rf $(BINDIR) # Package for Debian debpackage: diff --git a/README.md b/README.md index aa982db..019b8ff 100644 --- a/README.md +++ b/README.md @@ -575,3 +575,26 @@ See [configuration](./docs/gate-configuration.md). ## Credits Please see [CREDITS](CREDITS.md) for details. + +## Fuzzing + +To run fuzzing tests use the following command: + +```shell +$ make fuzz +``` + +This command will install dependencies for the fuzzing process and run existing fuzzing tests. + +You can also use the following arguments: + +``` +FUZZ_TIMEOUT - time to run each fuzzing test (default 30) +FUZZ_FUNCTIONS - fuzzing tests that will be started (default "all") +FUZZ_AUX - additional parameters for the fuzzer (for example, "-debug") +FUZZ_NGFUZZ_DIR - path to ngfuzz tool +```` + +## Credits + +Please see [CREDITS](CREDITS.md) for details. diff --git a/go.mod b/go.mod index 4e4bf9c..accedfb 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.13.0 + github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/valyala/fasthttp v1.34.0 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 diff --git a/go.sum b/go.sum index 73b2798..07eccfa 100644 --- a/go.sum +++ b/go.sum @@ -820,6 +820,8 @@ github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBN github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 h1:GpfJ7OdNjS7BFTVwNCUI9L4aCJOFRbr5fdHqjdhoYE8= +github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4/go.mod h1:f3jBhpWvuZmue0HZK52GzRHJOYHYSILs/c8+K2S/J+o= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= diff --git a/internal/handler/handler_fuzz_test.go b/internal/handler/handler_fuzz_test.go new file mode 100644 index 0000000..ad2ae6e --- /dev/null +++ b/internal/handler/handler_fuzz_test.go @@ -0,0 +1,580 @@ +//go:build gofuzz +// +build gofuzz + +package handler + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "mime/multipart" + "net/http" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + go_fuzz_utils "github.com/trailofbits/go-fuzz-utils" + "github.com/valyala/fasthttp" +) + +const ( + fuzzSuccessExitCode = 0 + fuzzFailExitCode = -1 +) + +func prepareStrings(tp *go_fuzz_utils.TypeProvider, count int) ([]string, error) { + array := make([]string, count) + var err error + + for i := 0; i < count; i++ { + err = tp.Reset() + if err != nil { + return nil, err + } + + array[i], err = tp.GetString() + if err != nil { + return nil, err + } + } + + return array, nil +} + +func prepareBools(tp *go_fuzz_utils.TypeProvider, count int) ([]bool, error) { + array := make([]bool, count) + var err error + + for i := 0; i < count; i++ { + err = tp.Reset() + if err != nil { + return nil, err + } + + array[i], err = tp.GetBool() + if err != nil { + return nil, err + } + } + + return array, nil +} + +func getRandomDeterministicPositiveIntInRange(tp *go_fuzz_utils.TypeProvider, max int) (int, error) { + count, err := tp.GetInt() + if err != nil { + return -1, err + } + count = count % max + if count < 0 { + count += max + } + return count, nil +} + +func generateHeaders(tp *go_fuzz_utils.TypeProvider, r *fasthttp.Request, params []string) error { + count, err := tp.GetInt() + if err != nil { + return err + } + count = count % len(params) + if count < 0 { + count += len(params) + } + + for i := 0; i < count; i++ { + position, err := tp.GetInt() + if err != nil { + return err + } + position = position % len(params) + if position < 0 { + position += len(params) + } + + v, err := tp.GetString() + if err != nil { + return err + } + + r.Header.Set(params[position], v) + + } + + return nil +} + +func maybeFillRandom(tp *go_fuzz_utils.TypeProvider, initValue string) (string, error) { + rnd, err := tp.GetBool() + if err != nil { + return "", err + } + if rnd == true { + initValue, err = tp.GetString() + if err != nil { + return "", err + } + } + return initValue, nil +} + +func upload(tp *go_fuzz_utils.TypeProvider) (context.Context, *handlerContext, cid.ID, *fasthttp.RequestCtx, string, string, string, error) { + hc, err := prepareHandlerContext() + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + aclList := []acl.Basic{ + acl.Private, + acl.PrivateExtended, + acl.PublicRO, + acl.PublicROExtended, + acl.PublicRW, + acl.PublicRWExtended, + acl.PublicAppend, + acl.PublicAppendExtended, + } + + pos, err := getRandomDeterministicPositiveIntInRange(tp, len(aclList)) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + acl := aclList[pos] + + strings, err := prepareStrings(tp, 6) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + bktName := strings[0] + objFileName := strings[1] + valAttr := strings[2] + keyAttr := strings[3] + + if len(bktName) == 0 { + return nil, nil, cid.ID{}, nil, "", "", "", errors.New("not enought buckets") + } + + cnrID, cnr, err := hc.prepareContainer(bktName, acl) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + hc.frostfs.SetContainer(cnrID, cnr) + + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cnrID.EncodeToString()) + + attributes := map[string]string{ + object.AttributeFileName: objFileName, + keyAttr: valAttr, + } + + var buff bytes.Buffer + w := multipart.NewWriter(&buff) + fw, err := w.CreateFormFile("file", attributes[object.AttributeFileName]) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + content, err := tp.GetBytes() + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + if _, err = io.Copy(fw, bytes.NewReader(content)); err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + if err = w.Close(); err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + r.Request.SetBodyStream(&buff, buff.Len()) + r.Request.Header.Set("Content-Type", w.FormDataContentType()) + r.Request.Header.Set("X-Attribute-"+keyAttr, valAttr) + + err = generateHeaders(tp, &r.Request, []string{"X-Attribute-", "X-Attribute-DupKey", "X-Attribute-MyAttribute", "X-Attribute-System-DupKey", "X-Attribute-System-Expiration-Epoch1", "X-Attribute-SYSTEM-Expiration-Epoch2", "X-Attribute-system-Expiration-Epoch3", "X-Attribute-User-Attribute", "X-Attribute-", "X-Attribute-FileName", "X-Attribute-FROSTFS", "X-Attribute-neofs", "X-Attribute-SYSTEM", "X-Attribute-System-Expiration-Duration", "X-Attribute-System-Expiration-Epoch", "X-Attribute-System-Expiration-RFC3339", "X-Attribute-System-Expiration-Timestamp", "X-Attribute-Timestamp", "X-Attribute-" + strings[4], "X-Attribute-System-" + strings[5]}) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + hc.Handler().Upload(r) + + if r.Response.StatusCode() != http.StatusOK { + return nil, nil, cid.ID{}, nil, "", "", "", errors.New("error on upload") + } + + return ctx, hc, cnrID, r, objFileName, keyAttr, valAttr, nil +} + +func InitFuzzUpload() { + +} + +func DoFuzzUpload(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + _, _, _, _, _, _, _, err = upload(tp) + if err != nil { + return fuzzFailExitCode + } + + return fuzzSuccessExitCode +} + +func FuzzUpload(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzUpload(data) + }) +} + +func downloadOrHead(tp *go_fuzz_utils.TypeProvider, ctx context.Context, hc *handlerContext, cnrID cid.ID, resp *fasthttp.RequestCtx, filename string) (*fasthttp.RequestCtx, error) { + + var putRes putResponse + + defer func() { + if r := recover(); r != nil { + panic(resp) + } + }() + + data := resp.Response.Body() + err := json.Unmarshal(data, &putRes) + + if err != nil { + return nil, err + } + + obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] + attr := object.NewAttribute() + attr.SetKey(object.AttributeFilePath) + + filename, err = maybeFillRandom(tp, filename) + if err != nil { + return nil, err + } + + attr.SetValue(filename) + obj.SetAttributes(append(obj.Attributes(), *attr)...) + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return nil, err + } + oid := putRes.ObjectID + oid, err = maybeFillRandom(tp, oid) + if err != nil { + return nil, err + } + r.SetUserValue("cid", cid) + r.SetUserValue("oid", oid) + + rnd, err := tp.GetBool() + if err != nil { + return nil, err + } + if rnd == true { + r.SetUserValue("download", "true") + } + + return r, nil +} + +func InitFuzzGet() { + +} + +func DoFuzzGet(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, resp, filename, _, _, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename) + if err != nil { + return fuzzFailExitCode + } + + hc.Handler().DownloadByAddressOrBucketName(r) + + return fuzzSuccessExitCode +} + +func FuzzGet(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzUpload(data) + }) +} + +func InitFuzzHead() { + +} + +func DoFuzzHead(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, resp, filename, _, _, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename) + if err != nil { + return fuzzFailExitCode + } + + hc.Handler().HeadByAddressOrBucketName(r) + + return fuzzSuccessExitCode +} + +func FuzzHead(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzHead(data) + }) +} + +func InitFuzzDownloadByAttribute() { + +} + +func DoFuzzDownloadByAttribute(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return fuzzFailExitCode + } + + attrKey, err = maybeFillRandom(tp, attrKey) + if err != nil { + return fuzzFailExitCode + } + + attrVal, err = maybeFillRandom(tp, attrVal) + if err != nil { + return fuzzFailExitCode + } + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cid) + r.SetUserValue("attr_key", attrKey) + r.SetUserValue("attr_val", attrVal) + + hc.Handler().DownloadByAttribute(r) + + return fuzzSuccessExitCode +} + +func FuzzDownloadByAttribute(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDownloadByAttribute(data) + }) +} + +func InitFuzzHeadByAttribute() { + +} + +func DoFuzzHeadByAttribute(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return fuzzFailExitCode + } + + attrKey, err = maybeFillRandom(tp, attrKey) + if err != nil { + return fuzzFailExitCode + } + + attrVal, err = maybeFillRandom(tp, attrVal) + if err != nil { + return fuzzFailExitCode + } + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cid) + r.SetUserValue("attr_key", attrKey) + r.SetUserValue("attr_val", attrVal) + + hc.Handler().HeadByAttribute(r) + + return fuzzSuccessExitCode +} + +func FuzzHeadByAttribute(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzHeadByAttribute(data) + }) +} + +func InitFuzzDownloadZipped() { + +} + +func DoFuzzDownloadZipped(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, _, _, _, _, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return fuzzFailExitCode + } + + prefix := "" + prefix, err = maybeFillRandom(tp, prefix) + if err != nil { + return fuzzFailExitCode + } + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cid) + r.SetUserValue("prefix", prefix) + + hc.Handler().DownloadZipped(r) + + return fuzzSuccessExitCode +} + +func FuzzDownloadZipped(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDownloadZipped(data) + }) +} + +func InitFuzzStoreBearerTokenAppCtx() { + +} + +func DoFuzzStoreBearerTokenAppCtx(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + prefix := "" + prefix, err = maybeFillRandom(tp, prefix) + if err != nil { + return fuzzFailExitCode + } + + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + + strings, err := prepareStrings(tp, 3) + + rand, err := prepareBools(tp, 2) + + if rand[0] == true { + r.Request.Header.Set(fasthttp.HeaderAuthorization, "Bearer"+strings[0]) + } else if rand[1] == true { + r.Request.Header.SetCookie(fasthttp.HeaderAuthorization, "Bearer"+strings[1]) + } else { + r.Request.Header.Set(fasthttp.HeaderAuthorization, "Bearer"+strings[0]) + r.Request.Header.SetCookie(fasthttp.HeaderAuthorization, "Bearer"+strings[1]) + } + + tokens.StoreBearerTokenAppCtx(ctx, r) + + return fuzzSuccessExitCode +} + +func FuzzStoreBearerTokenAppCtx(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzStoreBearerTokenAppCtx(data) + }) +} From 77ffde58e94ccc0ca9a7f20485c142b4d2769ff7 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Fri, 30 Aug 2024 13:02:49 +0300 Subject: [PATCH 04/86] [#123] Add SECURITY.md Signed-off-by: Pavel Pogodaev --- SECURITY.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..46fe535 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,26 @@ +# Security Policy + + +## How To Report a Vulnerability + +If you think you have found a vulnerability in this repository, please report it to us through coordinated disclosure. + +**Please do not report security vulnerabilities through public issues, discussions, or change requests.** + +Instead, you can report it using one of the following ways: + +* Contact the [TrueCloudLab Security Team](mailto:security@frostfs.info) via email + +Please include as much of the information listed below as you can to help us better understand and resolve the issue: + +* The type of issue (e.g., buffer overflow, or cross-site scripting) +* Affected version(s) +* Impact of the issue, including how an attacker might exploit the issue +* Step-by-step instructions to reproduce the issue +* The location of the affected source code (tag/branch/commit or direct URL) +* Full paths of source file(s) related to the manifestation of the issue +* Any special configuration required to reproduce the issue +* Any log files that are related to this issue (if possible) +* Proof-of-concept or exploit code (if possible) + +This information will help us triage your report more quickly. From 843708a558c5b9f308c8eafeaa09171a45500202 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Mon, 26 Aug 2024 18:47:33 +0300 Subject: [PATCH 05/86] [#134] Support percent-encoding Signed-off-by: Pavel Pogodaev --- CHANGELOG.md | 1 + internal/handler/handler.go | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b322c96..a489027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ This document outlines major changes between releases. ## [Unreleased] +- Support percent-encoding for GET queries (#134) ### Changed - Update go version to 1.22 (#132) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 0bbcdb9..4de9d9a 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -215,6 +215,12 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, log = h.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) ) + unescapedKey, err := url.QueryUnescape(key) + if err != nil { + logAndSendBucketError(req, log, err) + return + } + ctx := utils.GetContextFromRequest(req) bktInfo, err := h.getBucketInfo(ctx, bucketname, log) @@ -223,7 +229,7 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, return } - foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, key) + foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey) if err != nil { if errors.Is(err, tree.ErrNodeAccessDenied) { response.Error(req, "Access Denied", fasthttp.StatusForbidden) From 7e80f0cce6fba901414964347e0ccf7755ff8553 Mon Sep 17 00:00:00 2001 From: Aleksey Savaitan Date: Tue, 10 Sep 2024 10:09:51 +0300 Subject: [PATCH 06/86] [#139] Add root ca cert for telemetry configuration Signed-off-by: Aleksey Savaitan --- cmd/http-gw/app.go | 17 ++++++ cmd/http-gw/settings.go | 7 ++- config/config.env | 1 + config/config.yaml | 1 + docs/gate-configuration.md | 12 ++-- go.mod | 47 +++++++-------- go.sum | 119 ++++++++++++++++--------------------- 7 files changed, 102 insertions(+), 102 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 561598f..4c49ee4 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "crypto/x509" "errors" "fmt" "net/http" @@ -737,6 +738,22 @@ func (a *app) initTracing(ctx context.Context) { InstanceID: instanceID, Version: Version, } + + if trustedCa := a.cfg.GetString(cfgTracingTrustedCa); trustedCa != "" { + caBytes, err := os.ReadFile(trustedCa) + if err != nil { + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + return + } + certPool := x509.NewCertPool() + ok := certPool.AppendCertsFromPEM(caBytes) + if !ok { + a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert")) + return + } + cfg.ServerCaCertPool = certPool + } + updated, err := tracing.Setup(ctx, cfg) if err != nil { a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 0d97dcb..3fe0023 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -75,9 +75,10 @@ const ( cfgPprofAddress = "pprof.address" // Tracing ... - cfgTracingEnabled = "tracing.enabled" - cfgTracingExporter = "tracing.exporter" - cfgTracingEndpoint = "tracing.endpoint" + cfgTracingEnabled = "tracing.enabled" + cfgTracingExporter = "tracing.exporter" + cfgTracingEndpoint = "tracing.endpoint" + cfgTracingTrustedCa = "tracing.trusted_ca" // Pool config. cfgConTimeout = "connect_timeout" diff --git a/config/config.env b/config/config.env index 05b83b3..b7347d7 100644 --- a/config/config.env +++ b/config/config.env @@ -99,6 +99,7 @@ HTTP_GW_ZIP_COMPRESSION=false HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_EXPORTER="otlp_grpc" +HTTP_GW_TRACING_TRUSTED_CA="" HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/config.yaml b/config/config.yaml index 7f8077b..ef57612 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -13,6 +13,7 @@ tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost:4317" + trusted_ca: "" logger: level: debug # Log level. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 8e3daad..e93e6cf 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -256,13 +256,15 @@ tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost:4317" + trusted_ca: "/etc/ssl/telemetry-trusted-ca.pem" ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|------------|----------|---------------|------------------|---------------------------------------------------------------| -| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | -| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | -| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|--------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| +| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | +| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | +| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | +| `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | # `runtime` section Contains runtime parameters. diff --git a/go.mod b/go.mod index accedfb..3519a14 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 @@ -19,12 +19,12 @@ require ( github.com/testcontainers/testcontainers-go v0.13.0 github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/valyala/fasthttp v1.34.0 - go.opentelemetry.io/otel v1.16.0 - go.opentelemetry.io/otel/trace v1.16.0 + go.opentelemetry.io/otel v1.28.0 + go.opentelemetry.io/otel/trace v1.28.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/net v0.23.0 - google.golang.org/grpc v1.62.0 + golang.org/x/net v0.26.0 + google.golang.org/grpc v1.64.0 ) require ( @@ -39,7 +39,7 @@ require ( github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/containerd v1.6.2 // indirect @@ -51,16 +51,15 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/klauspost/compress v1.16.4 // indirect @@ -95,24 +94,22 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect go.etcd.io/bbolt v1.3.9 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect - go.opentelemetry.io/otel/metric v1.16.0 // indirect - go.opentelemetry.io/otel/sdk v1.16.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 07eccfa..bac691a 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 h1:MV/vKJWLQT34RRbXYvkNKFYGNjL5bRNuCQMXkbC7fLI= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36/go.mod h1:vluJ/+yQMcq8ZIZZSA7Te+JKClr0lgtRErjICvb8wto= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= @@ -137,8 +137,8 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0Bsq github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -160,11 +160,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= @@ -336,7 +332,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp/router v1.4.1 h1:3xPUO+hy/HAkgGDSd5sX5w18cyGDIFbC7vip8KwPDk8= @@ -366,8 +361,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -402,9 +397,6 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= -github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -436,8 +428,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -504,9 +494,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= @@ -739,8 +728,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -874,25 +863,23 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 h1:+XWJd3jf75RXJq29mxbuXhCXFDG3S3R4vBUeSI2P7tE= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0/go.mod h1:hqgzBPTf4yONMFgdZvL/bK42R/iinTyVQtiWihs3SZc= -go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= -go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= -go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= -go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -921,8 +908,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -959,8 +946,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1012,8 +999,8 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1023,7 +1010,6 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1035,8 +1021,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1127,23 +1113,22 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1213,8 +1198,8 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1290,13 +1275,10 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 h1:x9PwdEgd11LgK+orcck69WVRo7DezSO4VUMPI4xpc8A= -google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.mod h1:rbHMSEDyoYX62nRVLOCc4Qt1HbsdytAYoVwgjiOhF3I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c h1:NUsgEN92SQQqzfA+YtqYNqYmB3DMMYLlIwUZAQFVFbo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1320,9 +1302,8 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1336,8 +1317,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From a4233b006c6ce227078c05938c1f5a235933b7ab Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 23 Sep 2024 09:09:32 +0300 Subject: [PATCH 07/86] [#144] Update frostfs-sdk-go Signed-off-by: Nikita Zinkevich --- go.mod | 11 +++++++---- go.sum | 22 ++++++++++++++-------- internal/frostfs/frostfs.go | 5 ++++- metrics/metrics.go | 2 -- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 3519a14..afa704b 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.22 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 @@ -24,7 +24,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/net v0.26.0 - google.golang.org/grpc v1.64.0 + google.golang.org/grpc v1.66.2 ) require ( @@ -36,11 +36,12 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.9.2 // indirect + github.com/VictoriaMetrics/easyproto v0.1.4 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/containerd v1.6.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -62,8 +63,10 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.4 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/sys/mount v0.3.2 // indirect github.com/moby/sys/mountinfo v0.6.1 // indirect diff --git a/go.sum b/go.sum index bac691a..bc433eb 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 h1:XxvwQKJT/f16qS3df5PBQPRYKkhy0/A7zH6644QpKD0= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e h1:740ABnOBYx4o6jxULHdSSnVW2fYIO35ohg+Uz59sxd0= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 h1:MV/vKJWLQT34RRbXYvkNKFYGNjL5bRNuCQMXkbC7fLI= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36/go.mod h1:vluJ/+yQMcq8ZIZZSA7Te+JKClr0lgtRErjICvb8wto= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 h1:ijUci3thz0EwWkuRJDocW5D1RkVAJlt9xNG4CYepC90= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= @@ -101,6 +101,8 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/VictoriaMetrics/easyproto v0.1.4 h1:r8cNvo8o6sR4QShBXQd1bKw/VVLSQma/V2KhTBPf+Sc= +github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -142,8 +144,8 @@ github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= @@ -523,6 +525,8 @@ github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -564,6 +568,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -1302,8 +1308,8 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go index 2610564..6cf162a 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/frostfs/frostfs.go @@ -60,7 +60,10 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) } idObj, err := x.pool.PutObject(ctx, prmPut) - return idObj, handleObjectError("save object via connection pool", err) + if err != nil { + return oid.ID{}, handleObjectError("save object via connection pool", err) + } + return idObj.ObjectID, nil } // wraps io.ReadCloser and transforms Read errors related to access violation diff --git a/metrics/metrics.go b/metrics/metrics.go index bfb66ee..b516477 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -191,8 +191,6 @@ func (m *poolMetricsCollector) updateRequestsDuration(node pool.NodeStatistic) { m.requestDuration.WithLabelValues(node.Address(), methodGetContainer).Set(float64(node.AverageGetContainer().Milliseconds())) m.requestDuration.WithLabelValues(node.Address(), methodListContainer).Set(float64(node.AverageListContainer().Milliseconds())) m.requestDuration.WithLabelValues(node.Address(), methodDeleteContainer).Set(float64(node.AverageDeleteContainer().Milliseconds())) - m.requestDuration.WithLabelValues(node.Address(), methodGetContainerEacl).Set(float64(node.AverageGetContainerEACL().Milliseconds())) - m.requestDuration.WithLabelValues(node.Address(), methodSetContainerEacl).Set(float64(node.AverageSetContainerEACL().Milliseconds())) m.requestDuration.WithLabelValues(node.Address(), methodEndpointInfo).Set(float64(node.AverageEndpointInfo().Milliseconds())) m.requestDuration.WithLabelValues(node.Address(), methodNetworkInfo).Set(float64(node.AverageNetworkInfo().Milliseconds())) m.requestDuration.WithLabelValues(node.Address(), methodPutObject).Set(float64(node.AveragePutObject().Milliseconds())) From c8473498aeebf65e855badd995118893cbf5f6ba Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Wed, 25 Sep 2024 14:02:32 +0300 Subject: [PATCH 08/86] [#146] Fix of sighup traicing docs Signed-off-by: Roman Loginov --- docs/gate-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index e93e6cf..e0bb336 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -261,7 +261,7 @@ tracing: | Parameter | Type | SIGHUP reload | Default value | Description | |--------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| -| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | +| `enabled` | `bool` | no | `false` | Flag to enable the tracing. | | `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | | `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | | `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | From 77eb4745814640ff34a6b787712a427c1ff93519 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Thu, 26 Sep 2024 17:48:19 +0300 Subject: [PATCH 09/86] [#147] Add sampling configuration Signed-off-by: Pavel Pogodaev --- cmd/http-gw/settings.go | 73 +++++++++++++++++++++++++++----------- config/config.env | 6 +++- config/config.yaml | 5 +++ docs/gate-configuration.md | 17 ++++++--- 4 files changed, 75 insertions(+), 26 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 3fe0023..d9bbc53 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -41,6 +41,8 @@ const ( defaultConnectTimeout = 10 * time.Second defaultStreamTimeout = 10 * time.Second + defaultLoggerSamplerInterval = 1 * time.Second + defaultShutdownTimeout = 15 * time.Second defaultPoolErrorThreshold uint32 = 100 @@ -91,6 +93,11 @@ const ( cfgLoggerLevel = "logger.level" cfgLoggerDestination = "logger.destination" + cfgLoggerSamplingEnabled = "logger.sampling.enabled" + cfgLoggerSamplingInitial = "logger.sampling.initial" + cfgLoggerSamplingThereafter = "logger.sampling.thereafter" + cfgLoggerSamplingInterval = "logger.sampling.interval" + // Wallet. cfgWalletPassphrase = "wallet.passphrase" cfgWalletPath = "wallet.path" @@ -188,6 +195,10 @@ func settings() *viper.Viper { // logger: v.SetDefault(cfgLoggerLevel, "debug") v.SetDefault(cfgLoggerDestination, "stdout") + v.SetDefault(cfgLoggerSamplingEnabled, false) + v.SetDefault(cfgLoggerSamplingThereafter, 100) + v.SetDefault(cfgLoggerSamplingInitial, 100) + v.SetDefault(cfgLoggerSamplingInterval, defaultLoggerSamplerInterval) // pool: v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) @@ -386,9 +397,9 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { switch dest { case destinationStdout: - return newStdoutLogger(lvl) + return newStdoutLogger(v, lvl) case destinationJournald: - return newJournaldLogger(lvl) + return newJournaldLogger(v, lvl) default: panic(fmt.Sprintf("wrong destination for logger: %s", dest)) } @@ -405,39 +416,59 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { // Logger records a stack trace for all messages at or above fatal level. // // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. -func newStdoutLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { - c := zap.NewProductionConfig() - c.Level = zap.NewAtomicLevelAt(lvl) - c.Encoding = "console" - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder +func newStdoutLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { + stdout := zapcore.AddSync(os.Stderr) + level := zap.NewAtomicLevelAt(lvl) - l, err := c.Build( - zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), - ) - if err != nil { - panic(fmt.Sprintf("build zap logger instance: %v", err)) - } + consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level) + consoleOutCore = samplingEnabling(v, consoleOutCore) - return l, c.Level + l := zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) + return l, level } -func newJournaldLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { - c := zap.NewProductionConfig() - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - c.Level = zap.NewAtomicLevelAt(lvl) +func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { + level := zap.NewAtomicLevelAt(lvl) - encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields) + encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) - core := zapjournald.NewCore(c.Level, encoder, &journald.Journal{}, zapjournald.SyslogFields) + core := zapjournald.NewCore(level, encoder, &journald.Journal{}, zapjournald.SyslogFields) coreWithContext := core.With([]zapcore.Field{ zapjournald.SyslogFacility(zapjournald.LogDaemon), zapjournald.SyslogIdentifier(), zapjournald.SyslogPid(), }) + coreWithContext = samplingEnabling(v, coreWithContext) + l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) - return l, c.Level + return l, level +} + +func newLogEncoder() zapcore.Encoder { + c := zap.NewProductionEncoderConfig() + c.EncodeTime = zapcore.ISO8601TimeEncoder + + return zapcore.NewConsoleEncoder(c) +} + +func samplingEnabling(v *viper.Viper, core zapcore.Core) zapcore.Core { + // Zap samples by logging the first cgfLoggerSamplingInitial entries with a given level + // and message within the specified time interval. + // In the above config, only the first cgfLoggerSamplingInitial log entries with the same level and message + // are recorded in cfgLoggerSamplingInterval interval. Every other log entry will be dropped within the interval since + // cfgLoggerSamplingThereafter is specified here. + if v.GetBool(cfgLoggerSamplingEnabled) { + core = zapcore.NewSamplerWithOptions( + core, + v.GetDuration(cfgLoggerSamplingInterval), + v.GetInt(cfgLoggerSamplingInitial), + v.GetInt(cfgLoggerSamplingThereafter), + ) + } + + return core } func getLogLevel(v *viper.Viper) (zapcore.Level, error) { diff --git a/config/config.env b/config/config.env index b7347d7..e1d5e7d 100644 --- a/config/config.env +++ b/config/config.env @@ -14,8 +14,12 @@ HTTP_GW_PPROF_ADDRESS=localhost:8083 HTTP_GW_PROMETHEUS_ENABLED=true HTTP_GW_PROMETHEUS_ADDRESS=localhost:8084 -# Log level. +# Logger. HTTP_GW_LOGGER_LEVEL=debug +HTTP_GW_LOGGER_SAMPLING_ENABLED=false +HTTP_GW_LOGGER_SAMPLING_INITIAL=100 +HTTP_GW_LOGGER_SAMPLING_THEREAFTER=100 +HTTP_GW_LOGGER_SAMPLING_INTERVAL=1s HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443 HTTP_GW_SERVER_0_TLS_ENABLED=false diff --git a/config/config.yaml b/config/config.yaml index ef57612..1b87fe9 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -18,6 +18,11 @@ tracing: logger: level: debug # Log level. destination: stdout + sampling: + enabled: false + initial: 100 + thereafter: 100 + interval: 1s server: - address: 0.0.0.0:8080 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index e0bb336..fb5ad2f 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -164,12 +164,21 @@ server: logger: level: debug destination: stdout + sampling: + enabled: false + initial: 100 + thereafter: 100 + interval: 1s ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|---------------|----------|---------------|---------------|----------------------------------------------------------------------------------------------------| -| `level` | `string` | yes | `debug` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | -| `destination` | `string` | no | `stdout` | Destination for logger: `stdout` or `journald` | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------------|------------|---------------|---------------|----------------------------------------------------------------------------------------------------| +| `level` | `string` | yes | `debug` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | +| `destination` | `string` | no | `stdout` | Destination for logger: `stdout` or `journald` | +| `sampling.enabled` | `bool` | no | false | Sampling enabling flag. | +| `sampling.initial` | `int` | no | '100' | Sampling count of first log entries. | +| `sampling.thereafter` | `int` | no | '100' | Sampling count of entries after an `interval`. | +| `sampling.interval` | `duration` | no | '1s' | Sampling interval of messaging similar entries. | # `web` section From 8fe8f2dcc279548c6a8299fac5e3a3097ab9b20d Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Thu, 26 Sep 2024 17:32:27 +0300 Subject: [PATCH 10/86] [#137] Add index page support Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 54 ++++++ cmd/http-gw/settings.go | 6 + config/config.yaml | 7 +- docs/api.md | 13 +- docs/gate-configuration.md | 38 ++-- go.mod | 2 +- internal/api/tree.go | 1 + internal/frostfs/services/pool_wrapper.go | 84 ++++++++- internal/handler/browse.go | 157 +++++++++++++++++ internal/handler/handler.go | 59 +++++-- internal/handler/handler_test.go | 16 ++ internal/handler/reader.go | 7 + internal/handler/upload.go | 34 ++-- internal/handler/utils.go | 21 +++ internal/logs/logs.go | 2 + internal/templates/index.gotmpl | 90 ++++++++++ internal/templates/template.go | 6 + tokens/bearer-token.go | 4 +- tree/tree.go | 204 +++++++++++++++++++--- tree/tree_test.go | 13 +- 20 files changed, 738 insertions(+), 80 deletions(-) create mode 100644 internal/handler/browse.go create mode 100644 internal/templates/index.gotmpl create mode 100644 internal/templates/template.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 4c49ee4..8d5930d 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -6,6 +6,7 @@ import ( "crypto/x509" "errors" "fmt" + "io" "net/http" "os" "os/signal" @@ -23,6 +24,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" @@ -91,6 +93,8 @@ type ( defaultTimestamp bool zipCompression bool clientCut bool + returnIndexPage bool + indexPageTemplate string bufferMaxSizeForPut uint64 namespaceHeader string defaultNamespaces []string @@ -155,6 +159,7 @@ func newApp(ctx context.Context, opt ...Option) App { a.initResolver() a.initMetrics() a.initTracing(ctx) + a.loadIndexPageTemplate() return a } @@ -177,12 +182,59 @@ func (s *appSettings) ZipCompression() bool { return s.zipCompression } +func (s *appSettings) IndexPageEnabled() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.returnIndexPage +} + +func (s *appSettings) IndexPageTemplate() string { + s.mu.RLock() + defer s.mu.RUnlock() + if s.indexPageTemplate == "" { + return templates.DefaultIndexTemplate + } + return s.indexPageTemplate +} + func (s *appSettings) setZipCompression(val bool) { s.mu.Lock() s.zipCompression = val s.mu.Unlock() } +func (s *appSettings) setReturnIndexPage(val bool) { + s.mu.Lock() + s.returnIndexPage = val + s.mu.Unlock() +} + +func (s *appSettings) setIndexTemplate(val string) { + s.mu.Lock() + s.indexPageTemplate = val + s.mu.Unlock() +} + +func (a *app) loadIndexPageTemplate() { + if !a.settings.IndexPageEnabled() { + return + } + reader, err := os.Open(a.cfg.GetString(cfgIndexPageTemplatePath)) + if err != nil { + a.settings.setIndexTemplate("") + a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + return + } + tmpl, err := io.ReadAll(reader) + if err != nil { + a.settings.setIndexTemplate("") + a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + return + } + a.settings.setIndexTemplate(string(tmpl)) + a.log.Info(logs.SetCustomIndexPageTemplate) +} + func (s *appSettings) ClientCut() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -491,6 +543,7 @@ func (a *app) configReload(ctx context.Context) { a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) a.initTracing(ctx) + a.loadIndexPageTemplate() a.setHealthStatus() a.log.Info(logs.SIGHUPConfigReloadCompleted) @@ -499,6 +552,7 @@ func (a *app) configReload(ctx context.Context) { func (a *app) updateSettings() { a.settings.setDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) a.settings.setZipCompression(a.cfg.GetBool(cfgZipCompression)) + a.settings.setReturnIndexPage(a.cfg.GetBool(cfgIndexPageEnabled)) a.settings.setClientCut(a.cfg.GetBool(cfgClientCut)) a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut)) a.settings.setNamespaceHeader(a.cfg.GetString(cfgResolveNamespaceHeader)) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index d9bbc53..eab5b6b 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -62,6 +62,9 @@ const ( cfgReconnectInterval = "reconnect_interval" + cfgIndexPageEnabled = "index_page.enabled" + cfgIndexPageTemplatePath = "index_page.template_path" + // Web. cfgWebReadBufferSize = "web.read_buffer_size" cfgWebWriteBufferSize = "web.write_buffer_size" @@ -203,6 +206,9 @@ func settings() *viper.Viper { // pool: v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) + v.SetDefault(cfgIndexPageEnabled, false) + v.SetDefault(cfgIndexPageTemplatePath, "") + // frostfs: v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) diff --git a/config/config.yaml b/config/config.yaml index 1b87fe9..61aa70b 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -107,6 +107,11 @@ request_timeout: 5s # Timeout to check node health during rebalance. rebalance_timer: 30s # Interval to check nodes health. pool_error_threshold: 100 # The number of errors on connection after which node is considered as unhealthy. +# Enable index page to see objects list for specified container and prefix +index_page: + enabled: false + template_path: internal/handler/templates/index.gotmpl + zip: compression: false # Enable zip compression to download files by common prefix. @@ -132,4 +137,4 @@ cache: resolve_bucket: namespace_header: X-Frostfs-Namespace - default_namespaces: [ "", "root" ] \ No newline at end of file + default_namespaces: [ "", "root" ] diff --git a/docs/api.md b/docs/api.md index 78df766..f7eb3a4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -95,12 +95,12 @@ The `filename` field from the multipart form will be set as `FileName` attribute ## Get object -Route: `/get/{cid}/{oid}?[download=true]` +Route: `/get/{cid}/{oid}?[download=false]` | Route parameter | Type | Description | |-----------------|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `cid` | Single | Base58 encoded container ID or container name from NNS. | -| `oid` | Single | Base58 encoded object ID. | +| `cid` | Single | Base58 encoded `container ID` or `container name` from NNS or `bucket name`. | +| `oid` | Single | Base58 encoded `object ID`. Also could be `S3 object name` if `cid` is specified as bucket name. | | `download` | Query | Set the `Content-Disposition` header as `attachment` in response.
This make the browser to download object as file instead of showing it on the page. | ### Methods @@ -141,6 +141,13 @@ Get an object (payload and attributes) by an address. | 400 | Some error occurred during object downloading. | | 404 | Container or object not found. | +###### Body + +Returns object data. If request performed from browser, either displays raw data or downloads it as +attachment if `download` query parameter is set to `true`. +If `index_page.enabled` is set to `true`, returns HTML with index-page if no object with specified +S3-name was found. + #### HEAD Get an object attributes by an address. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index fb5ad2f..e8d1f4b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -57,6 +57,7 @@ $ cat http.log | `frostfs` | [Frostfs configuration](#frostfs-section) | | `cache` | [Cache configuration](#cache-section) | | `resolve_bucket` | [Bucket name resolving configuration](#resolve_bucket-section) | +| `index_page` | [Index page configuration](#index_page-section) | # General section @@ -75,16 +76,16 @@ pool_error_threshold: 100 reconnect_interval: 1m ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|------------------------|------------|---------------|----------------|------------------------------------------------------------------------------------| -| `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | -| `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | -| `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | -| `stream_timeout` | `duration` | | `10s` | Timeout for individual operations in streaming RPC. | -| `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | -| `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | -| `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | -| `reconnect_interval` | `duration` | no | `1m` | Listeners reconnection interval. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|------------------------|------------|---------------|---------------|-------------------------------------------------------------------------------------------------| +| `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | +| `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | +| `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | +| `stream_timeout` | `duration` | | `10s` | Timeout for individual operations in streaming RPC. | +| `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | +| `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | +| `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | +| `reconnect_interval` | `duration` | no | `1m` | Listeners reconnection interval. | # `wallet` section @@ -346,4 +347,19 @@ resolve_bucket: | Parameter | Type | SIGHUP reload | Default value | Description | |----------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------| | `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. | -| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | \ No newline at end of file +| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | + +# `index_page` section + +Parameters for index HTML-page output with S3-bucket or S3-subdir content for `Get object` request + +```yaml +index_page: + enabled: false + template_path: "" +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------|----------|---------------|---------------|---------------------------------------------------------------------------------| +| `enabled` | `bool` | yes | `false` | Flag to enable index_page return if no object with specified S3-name was found. | +| `template_path` | `string` | yes | `""` | Path to .gotmpl file with html template for index_page. | diff --git a/go.mod b/go.mod index afa704b..d1a3788 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 + github.com/docker/go-units v0.4.0 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.106.2 github.com/prometheus/client_golang v1.19.0 @@ -50,7 +51,6 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/docker v20.10.14+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect - github.com/docker/go-units v0.4.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/internal/api/tree.go b/internal/api/tree.go index 4d16cc7..5b1d608 100644 --- a/internal/api/tree.go +++ b/internal/api/tree.go @@ -8,6 +8,7 @@ import ( type NodeVersion struct { BaseNodeVersion DeleteMarker bool + IsPrefixNode bool } // BaseNodeVersion is minimal node info from tree service. diff --git a/internal/frostfs/services/pool_wrapper.go b/internal/frostfs/services/pool_wrapper.go index f7b0a26..fa70f15 100644 --- a/internal/frostfs/services/pool_wrapper.go +++ b/internal/frostfs/services/pool_wrapper.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "io" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" @@ -15,16 +17,16 @@ type GetNodeByPathResponseInfoWrapper struct { response *grpcService.GetNodeByPathResponse_Info } -func (n GetNodeByPathResponseInfoWrapper) GetNodeID() uint64 { - return n.response.GetNodeId() +func (n GetNodeByPathResponseInfoWrapper) GetNodeID() []uint64 { + return []uint64{n.response.GetNodeId()} } -func (n GetNodeByPathResponseInfoWrapper) GetParentID() uint64 { - return n.response.GetParentId() +func (n GetNodeByPathResponseInfoWrapper) GetParentID() []uint64 { + return []uint64{n.response.GetParentId()} } -func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() uint64 { - return n.response.GetTimestamp() +func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() []uint64 { + return []uint64{n.response.GetTimestamp()} } func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { @@ -89,3 +91,73 @@ func handleError(err error) error { return err } + +func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) { + order := treepool.NoneOrder + if sort { + order = treepool.AscendingOrder + } + poolPrm := treepool.GetSubTreeParams{ + CID: bktInfo.CID, + TreeID: treeID, + RootID: rootID, + Depth: depth, + BearerToken: getBearer(ctx), + Order: order, + } + if len(rootID) == 1 && rootID[0] == 0 { + // storage node interprets 'nil' value as []uint64{0} + // gate wants to send 'nil' value instead of []uint64{0}, because + // it provides compatibility with previous tree service api where + // single uint64(0) value is dropped from signature + poolPrm.RootID = nil + } + + subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) + if err != nil { + return nil, handleError(err) + } + + var subtree []tree.NodeResponse + + node, err := subTreeReader.Next() + for err == nil { + subtree = append(subtree, GetSubTreeResponseBodyWrapper{node}) + node, err = subTreeReader.Next() + } + if err != io.EOF { + return nil, handleError(err) + } + + return subtree, nil +} + +type GetSubTreeResponseBodyWrapper struct { + response *grpcService.GetSubTreeResponse_Body +} + +func (n GetSubTreeResponseBodyWrapper) GetNodeID() []uint64 { + return n.response.GetNodeId() +} + +func (n GetSubTreeResponseBodyWrapper) GetParentID() []uint64 { + resp := n.response.GetParentId() + if resp == nil { + // storage sends nil that should be interpreted as []uint64{0} + // due to protobuf compatibility, see 'GetSubTree' function + return []uint64{0} + } + return resp +} + +func (n GetSubTreeResponseBodyWrapper) GetTimestamp() []uint64 { + return n.response.GetTimestamp() +} + +func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.response.Meta)) + for i, value := range n.response.Meta { + res[i] = value + } + return res +} diff --git a/internal/handler/browse.go b/internal/handler/browse.go new file mode 100644 index 0000000..e84fb04 --- /dev/null +++ b/internal/handler/browse.go @@ -0,0 +1,157 @@ +package handler + +import ( + "html/template" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "github.com/docker/go-units" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +const ( + dateFormat = "02-01-2006 15:04" + attrOID = "OID" + attrCreated = "Created" + attrFileName = "FileName" + attrSize = "Size" +) + +type ( + BrowsePageData struct { + BucketName, + Prefix string + Objects []ResponseObject + } + ResponseObject struct { + OID string + Created string + FileName string + Size string + IsDir bool + } +) + +func parseTimestamp(tstamp string) (time.Time, error) { + millis, err := strconv.ParseInt(tstamp, 10, 64) + if err != nil { + return time.Time{}, err + } + + return time.UnixMilli(millis), nil +} + +func NewResponseObject(nodes map[string]string) ResponseObject { + return ResponseObject{ + OID: nodes[attrOID], + Created: nodes[attrCreated], + FileName: nodes[attrFileName], + Size: nodes[attrSize], + IsDir: nodes[attrOID] == "", + } +} + +func formatTimestamp(strdate string) string { + date, err := parseTimestamp(strdate) + if err != nil || date.IsZero() { + return "" + } + + return date.Format(dateFormat) +} + +func formatSize(strsize string) string { + size, err := strconv.ParseFloat(strsize, 64) + if err != nil { + return "0B" + } + return units.HumanSize(size) +} + +func parentDir(prefix string) string { + index := strings.LastIndex(prefix, "/") + if index == -1 { + return prefix + } + return prefix[index:] +} + +func trimPrefix(encPrefix string) string { + prefix, err := url.PathUnescape(encPrefix) + if err != nil { + return "" + } + slashIndex := strings.LastIndex(prefix, "/") + if slashIndex == -1 { + return "" + } + return prefix[:slashIndex] +} + +func urlencode(prefix, filename string) string { + var res strings.Builder + path := filename + if prefix != "" { + path = strings.Join([]string{prefix, filename}, "/") + } + prefixParts := strings.Split(path, "/") + for _, prefixPart := range prefixParts { + prefixPart = "/" + url.PathEscape(prefixPart) + if prefixPart == "/." || prefixPart == "/.." { + prefixPart = url.PathEscape(prefixPart) + } + res.WriteString(prefixPart) + } + + return res.String() +} + +func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) { + log := h.log.With(zap.String("bucket", bucketInfo.Name)) + ctx := utils.GetContextFromRequest(c) + nodes, err := h.listObjects(ctx, bucketInfo, prefix) + if err != nil { + logAndSendBucketError(c, log, err) + return + } + + respObjects := make([]ResponseObject, len(nodes)) + + for i, node := range nodes { + respObjects[i] = NewResponseObject(node) + } + + sort.Slice(respObjects, func(i, j int) bool { + if respObjects[i].IsDir == respObjects[j].IsDir { + return respObjects[i].FileName < respObjects[j].FileName + } + return respObjects[i].IsDir + }) + indexTemplate := h.config.IndexPageTemplate() + + tmpl, err := template.New("index").Funcs(template.FuncMap{ + "formatTimestamp": formatTimestamp, + "formatSize": formatSize, + "trimPrefix": trimPrefix, + "urlencode": urlencode, + "parentDir": parentDir, + }).Parse(indexTemplate) + if err != nil { + logAndSendBucketError(c, log, err) + return + } + if err = tmpl.Execute(c, &BrowsePageData{ + BucketName: bucketInfo.Name, + Prefix: prefix, + Objects: respObjects, + }); err != nil { + logAndSendBucketError(c, log, err) + return + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4de9d9a..8b07af0 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -30,6 +30,8 @@ type Config interface { DefaultTimestamp() bool ZipCompression() bool ClientCut() bool + IndexPageEnabled() bool + IndexPageTemplate() string BufferMaxSizeForPut() uint64 NamespaceHeader() string } @@ -208,41 +210,50 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ // byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { var ( - bucketname = req.UserValue("cid").(string) - key = req.UserValue("oid").(string) + bucketname = c.UserValue("cid").(string) + key = c.UserValue("oid").(string) log = h.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) + download = c.QueryArgs().GetBool("download") ) unescapedKey, err := url.QueryUnescape(key) if err != nil { - logAndSendBucketError(req, log, err) + logAndSendBucketError(c, log, err) return } - ctx := utils.GetContextFromRequest(req) + ctx := utils.GetContextFromRequest(c) bktInfo, err := h.getBucketInfo(ctx, bucketname, log) if err != nil { - logAndSendBucketError(req, log, err) + logAndSendBucketError(c, log, err) return } foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey) + if h.config.IndexPageEnabled() && !download && string(c.Method()) != fasthttp.MethodHead { + if isDir(unescapedKey) || isContainerRoot(unescapedKey) { + if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK { + c.SetStatusCode(code) + h.browseObjects(c, bktInfo, unescapedKey) + return + } + } + } if err != nil { if errors.Is(err, tree.ErrNodeAccessDenied) { - response.Error(req, "Access Denied", fasthttp.StatusForbidden) - return + response.Error(c, "Access Denied", fasthttp.StatusForbidden) + } else { + response.Error(c, "object wasn't found", fasthttp.StatusNotFound) + log.Error(logs.GetLatestObjectVersion, zap.Error(err)) } - log.Error(logs.GetLatestObjectVersion, zap.Error(err)) - - response.Error(req, "object wasn't found", fasthttp.StatusNotFound) return } if foundOid.DeleteMarker { log.Error(logs.ObjectWasDeleted) - response.Error(req, "object deleted", fasthttp.StatusNotFound) + response.Error(c, "object deleted", fasthttp.StatusNotFound) return } @@ -250,7 +261,7 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, addr.SetContainer(bktInfo.CID) addr.SetObject(foundOid.OID) - f(ctx, *h.newRequest(req, log), addr) + f(ctx, *h.newRequest(c, log), addr) } // byAttribute is a wrapper similar to byAddress. @@ -379,3 +390,25 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } + +func (h *Handler) listObjects(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]map[string]string, error) { + nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) + if err != nil { + return nil, err + } + + var objects = make([]map[string]string, 0, len(nodes)) + for _, node := range nodes { + meta := node.GetMeta() + if meta == nil { + continue + } + var obj = make(map[string]string, len(meta)) + for _, m := range meta { + obj[m.GetKey()] = string(m.GetValue()) + } + objects = append(objects, obj) + } + + return objects, nil +} diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index ed67f88..4fe9153 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -12,6 +12,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" @@ -37,6 +38,10 @@ func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree return nil, nil } +func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) { + return nil, nil +} + type configMock struct { } @@ -48,6 +53,17 @@ func (c *configMock) ZipCompression() bool { return false } +func (c *configMock) IndexPageEnabled() bool { + return false +} + +func (c *configMock) IndexPageTemplatePath() string { + return "" +} +func (c *configMock) IndexPageTemplate() string { + return "" +} + func (c *configMock) ClientCut() bool { return false } diff --git a/internal/handler/reader.go b/internal/handler/reader.go index 65d258b..d901f25 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -53,6 +53,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi dis = "inline" start = time.Now() filename string + filepath string ) prm := PrmObjectGet{ @@ -104,6 +105,8 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val + case object.AttributeFilePath: + filepath = val } } @@ -135,6 +138,10 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi } req.SetContentType(contentType) + if filename == "" { + filename = filepath + } + req.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) req.Response.SetBodyStream(rObj.Payload, int(payloadSize)) diff --git a/internal/handler/upload.go b/internal/handler/upload.go index cea2250..6c0e117 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -43,22 +43,22 @@ func (pr *putResponse) encode(w io.Writer) error { } // Upload handles multipart upload request. -func (h *Handler) Upload(req *fasthttp.RequestCtx) { +func (h *Handler) Upload(c *fasthttp.RequestCtx) { var ( file MultipartFile idObj oid.ID addr oid.Address - scid, _ = req.UserValue("cid").(string) + scid, _ = c.UserValue("cid").(string) log = h.log.With(zap.String("cid", scid)) - bodyStream = req.RequestBodyStream() + bodyStream = c.RequestBodyStream() drainBuf = make([]byte, drainBufSize) ) - ctx := utils.GetContextFromRequest(req) + ctx := utils.GetContextFromRequest(c) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { - logAndSendBucketError(req, log, err) + logAndSendBucketError(c, log, err) return } @@ -75,21 +75,21 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { zap.Error(err), ) }() - boundary := string(req.Request.Header.MultipartFormBoundary()) + boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(h.log, bodyStream, boundary); err != nil { log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) - response.Error(req, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) + response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } - filtered, err := filterHeaders(h.log, &req.Request.Header) + filtered, err := filterHeaders(h.log, &c.Request.Header) if err != nil { log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) - response.Error(req, err.Error(), fasthttp.StatusBadRequest) + response.Error(c, err.Error(), fasthttp.StatusBadRequest) return } now := time.Now() - if rawHeader := req.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { + if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err)) } else { @@ -97,9 +97,9 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } } - if err = utils.PrepareExpirationHeader(req, h.frostfs, filtered, now); err != nil { + if err = utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) - response.Error(req, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) + response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -143,7 +143,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } if idObj, err = h.frostfs.CreateObject(ctx, prm); err != nil { - h.handlePutFrostFSErr(req, err) + h.handlePutFrostFSErr(c, err) return } @@ -151,9 +151,9 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { addr.SetContainer(bktInfo.CID) // Try to return the response, otherwise, if something went wrong, throw an error. - if err = newPutResponse(addr).encode(req); err != nil { + if err = newPutResponse(addr).encode(c); err != nil { log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) - response.Error(req, "could not encode response", fasthttp.StatusBadRequest) + response.Error(c, "could not encode response", fasthttp.StatusBadRequest) return } @@ -170,8 +170,8 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } } // Report status code and content type. - req.Response.SetStatusCode(fasthttp.StatusOK) - req.Response.Header.SetContentType(jsonHeader) + c.Response.SetStatusCode(fasthttp.StatusOK) + c.Response.Header.SetContentType(jsonHeader) } func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index a5a53ed..7fa1158 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -2,12 +2,14 @@ package handler import ( "context" + "errors" "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/valyala/fasthttp" @@ -38,6 +40,25 @@ func bearerToken(ctx context.Context) *bearer.Token { return nil } +func isDir(name string) bool { + return strings.HasSuffix(name, "/") +} + +func isContainerRoot(key string) bool { + return key == "" +} + +func checkErrorType(err error) int { + switch { + case err == nil: + return fasthttp.StatusOK + case errors.Is(err, tree.ErrNodeAccessDenied): + return fasthttp.StatusForbidden + default: + return fasthttp.StatusNotFound + } +} + func isValidToken(s string) bool { for _, c := range s { if c <= ' ' || c > 127 { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 0ab5dbf..96bdaa5 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -31,6 +31,8 @@ const ( CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go + FailedToReadIndexPageTemplate = "failed to read index page template, set default" // Warn in ../../app.go + SetCustomIndexPageTemplate = "set custom index page template" // Info in ../../app.go ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go diff --git a/internal/templates/index.gotmpl b/internal/templates/index.gotmpl new file mode 100644 index 0000000..ea66a62 --- /dev/null +++ b/internal/templates/index.gotmpl @@ -0,0 +1,90 @@ +{{$bucketName := .BucketName}} +{{ $prefix := trimPrefix .Prefix }} + + + + + Index of s3://{{$bucketName}}/{{if $prefix}}/{{$prefix}}/{{end}} + + + +

Index of s3://{{$bucketName}}/{{if $prefix}}{{$prefix}}/{{end}}

+ + + + + + + + + + + {{ $trimmedPrefix := trimPrefix $prefix }} + {{if $trimmedPrefix }} + + + + + + + {{else}} + + + + + + + {{end}} + {{range .Objects}} + + + + + + + {{end}} + +
FilenameSizeCreatedDownload
+ ⮐.. +
+ ⮐.. +
+ {{if .IsDir}} + 🗀 + + {{.FileName}}/ + + {{else}} + 🗎 + + {{.FileName}} + + {{end}} + {{if not .IsDir}}{{ formatSize .Size }}{{end}}{{if not .IsDir}}{{ formatTimestamp .Created }}{{end}} + {{ if not .IsDir }} + + Link + + {{ end }} +
+ + diff --git a/internal/templates/template.go b/internal/templates/template.go new file mode 100644 index 0000000..b9885e6 --- /dev/null +++ b/internal/templates/template.go @@ -0,0 +1,6 @@ +package templates + +import _ "embed" + +//go:embed index.gotmpl +var DefaultIndexTemplate string diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index b01860d..880a100 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -52,8 +52,8 @@ func BearerTokenFromCookie(h *fasthttp.RequestHeader) []byte { // StoreBearerTokenAppCtx extracts a bearer token from the header or cookie and stores // it in the application context. -func StoreBearerTokenAppCtx(ctx context.Context, req *fasthttp.RequestCtx) (context.Context, error) { - tkn, err := fetchBearerToken(req) +func StoreBearerTokenAppCtx(ctx context.Context, c *fasthttp.RequestCtx) (context.Context, error) { + tkn, err := fetchBearerToken(c) if err != nil { return nil, err } diff --git a/tree/tree.go b/tree/tree.go index a9135eb..162f41f 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -2,11 +2,13 @@ package tree import ( "context" + "errors" "fmt" "strings" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -20,6 +22,7 @@ type ( // Each method must return ErrNodeNotFound or ErrNodeAccessDenied if relevant. ServiceClient interface { GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error) + GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]NodeResponse, error) } treeNode struct { @@ -29,6 +32,7 @@ type ( GetNodesParams struct { CnrID cid.ID + BktInfo *data.BucketInfo TreeID string Path []string Meta []string @@ -54,6 +58,7 @@ const ( // keys for delete marker nodes. isDeleteMarkerKV = "IsDeleteMarker" + sizeKV = "Size" // versionTree -- ID of a tree with object versions. versionTree = "version" @@ -73,26 +78,28 @@ type Meta interface { type NodeResponse interface { GetMeta() []Meta - GetTimestamp() uint64 + GetTimestamp() []uint64 + GetNodeID() []uint64 + GetParentID() []uint64 } func newTreeNode(nodeInfo NodeResponse) (*treeNode, error) { - treeNode := &treeNode{ + tNode := &treeNode{ Meta: make(map[string]string, len(nodeInfo.GetMeta())), } for _, kv := range nodeInfo.GetMeta() { switch kv.GetKey() { case oidKV: - if err := treeNode.ObjID.DecodeString(string(kv.GetValue())); err != nil { + if err := tNode.ObjID.DecodeString(string(kv.GetValue())); err != nil { return nil, err } default: - treeNode.Meta[kv.GetKey()] = string(kv.GetValue()) + tNode.Meta[kv.GetKey()] = string(kv.GetValue()) } } - return treeNode, nil + return tNode, nil } func (n *treeNode) Get(key string) (string, bool) { @@ -106,29 +113,44 @@ func (n *treeNode) FileName() (string, bool) { } func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { - treeNode, err := newTreeNode(node) + tNode, err := newTreeNode(node) if err != nil { return nil, fmt.Errorf("invalid tree node: %w", err) } - return newNodeVersionFromTreeNode(treeNode), nil + return newNodeVersionFromTreeNode(tNode), nil } func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) - + size, _ := treeNode.Get(sizeKV) version := &api.NodeVersion{ BaseNodeVersion: api.BaseNodeVersion{ OID: treeNode.ObjID, }, DeleteMarker: isDeleteMarker, + IsPrefixNode: size == "", } return version } func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { - meta := []string{oidKV, isDeleteMarkerKV} + nodes, err := c.GetVersions(ctx, cnrID, objectName) + if err != nil { + return nil, err + } + + latestNode, err := getLatestVersionNode(nodes) + if err != nil { + return nil, err + } + + return newNodeVersion(latestNode) +} + +func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) { + meta := []string{oidKV, isDeleteMarkerKV, sizeKV} path := pathFromName(objectName) p := &GetNodesParams{ @@ -139,30 +161,24 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s LatestOnly: false, AllAttrs: false, } - nodes, err := c.service.GetNodes(ctx, p) - if err != nil { - return nil, err - } - latestNode, err := getLatestNode(nodes) - if err != nil { - return nil, err - } - - return newNodeVersion(latestNode) + return c.service.GetNodes(ctx, p) } -func getLatestNode(nodes []NodeResponse) (NodeResponse, error) { +func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { var ( maxCreationTime uint64 targetIndexNode = -1 ) for i, node := range nodes { - currentCreationTime := node.GetTimestamp() - if checkExistOID(node.GetMeta()) && currentCreationTime > maxCreationTime { - maxCreationTime = currentCreationTime + if !checkExistOID(node.GetMeta()) { + continue + } + + if currentCreationTime := getMaxTimestamp(node); currentCreationTime > maxCreationTime { targetIndexNode = i + maxCreationTime = currentCreationTime } } @@ -187,3 +203,145 @@ func checkExistOID(meta []Meta) bool { func pathFromName(objectName string) []string { return strings.Split(objectName, separator) } + +func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]NodeResponse, string, error) { + rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) + if err != nil { + return nil, "", err + } + subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) + if err != nil { + if errors.Is(err, layer.ErrNodeNotFound) { + return nil, "", nil + } + return nil, "", err + } + + nodesMap := make(map[string][]NodeResponse, len(subTree)) + for _, node := range subTree { + if MultiID(rootID).Equal(node.GetNodeID()) { + continue + } + + fileName := GetFilename(node) + if !strings.HasPrefix(fileName, tailPrefix) { + continue + } + + nodes := nodesMap[fileName] + + // Add all nodes if flag latestOnly is false. + // Add all intermediate nodes + // and only latest leaf (object) nodes. To do this store and replace last leaf (object) node in nodes[0] + if len(nodes) == 0 { + nodes = []NodeResponse{node} + } else if !latestOnly || isIntermediate(node) { + nodes = append(nodes, node) + } else if isIntermediate(nodes[0]) { + nodes = append([]NodeResponse{node}, nodes...) + } else if getMaxTimestamp(node) > getMaxTimestamp(nodes[0]) { + nodes[0] = node + } + + nodesMap[fileName] = nodes + } + + result := make([]NodeResponse, 0, len(subTree)) + for _, nodes := range nodesMap { + result = append(result, nodes...) + } + + return result, strings.TrimSuffix(prefix, tailPrefix), nil +} + +func (c *Tree) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) ([]uint64, string, error) { + rootID := []uint64{0} + path := strings.Split(prefix, separator) + tailPrefix := path[len(path)-1] + + if len(path) > 1 { + var err error + rootID, err = c.getPrefixNodeID(ctx, bktInfo, treeID, path[:len(path)-1]) + if err != nil { + return nil, "", err + } + } + + return rootID, tailPrefix, nil +} + +func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefixPath []string) ([]uint64, error) { + p := &GetNodesParams{ + CnrID: bktInfo.CID, + BktInfo: bktInfo, + TreeID: treeID, + Path: prefixPath, + LatestOnly: false, + AllAttrs: true, + } + nodes, err := c.service.GetNodes(ctx, p) + if err != nil { + return nil, err + } + + var intermediateNodes []uint64 + for _, node := range nodes { + if isIntermediate(node) { + intermediateNodes = append(intermediateNodes, node.GetNodeID()...) + } + } + + if len(intermediateNodes) == 0 { + return nil, layer.ErrNodeNotFound + } + + return intermediateNodes, nil +} + +func GetFilename(node NodeResponse) string { + for _, kv := range node.GetMeta() { + if kv.GetKey() == FileNameKey { + return string(kv.GetValue()) + } + } + + return "" +} + +func isIntermediate(node NodeResponse) bool { + if len(node.GetMeta()) != 1 { + return false + } + + return node.GetMeta()[0].GetKey() == FileNameKey +} + +func getMaxTimestamp(node NodeResponse) uint64 { + var maxTimestamp uint64 + + for _, timestamp := range node.GetTimestamp() { + if timestamp > maxTimestamp { + maxTimestamp = timestamp + } + } + + return maxTimestamp +} + +type MultiID []uint64 + +func (m MultiID) Equal(id MultiID) bool { + seen := make(map[uint64]struct{}, len(m)) + + for i := range m { + seen[m[i]] = struct{}{} + } + + for i := range id { + if _, ok := seen[id[i]]; !ok { + return false + } + } + + return true +} diff --git a/tree/tree_test.go b/tree/tree_test.go index 7cd2314..69ac5f6 100644 --- a/tree/tree_test.go +++ b/tree/tree_test.go @@ -24,8 +24,8 @@ type nodeResponse struct { timestamp uint64 } -func (n nodeResponse) GetTimestamp() uint64 { - return n.timestamp +func (n nodeResponse) GetTimestamp() []uint64 { + return []uint64{n.timestamp} } func (n nodeResponse) GetMeta() []Meta { @@ -36,6 +36,13 @@ func (n nodeResponse) GetMeta() []Meta { return res } +func (n nodeResponse) GetNodeID() []uint64 { + return nil +} +func (n nodeResponse) GetParentID() []uint64 { + return nil +} + func TestGetLatestNode(t *testing.T) { for _, tc := range []struct { name string @@ -130,7 +137,7 @@ func TestGetLatestNode(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - actualNode, err := getLatestNode(tc.nodes) + actualNode, err := getLatestVersionNode(tc.nodes) if tc.error { require.Error(t, err) return From 495f7455355c11d3f4262ea61044a22bd45c301a Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Wed, 18 Sep 2024 07:35:26 +0300 Subject: [PATCH 11/86] [#142] Fix multipart-objects download Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 5 +- internal/handler/frostfs_mock.go | 11 +- internal/handler/handler.go | 19 +- internal/handler/multipart.go | 31 +++ internal/handler/reader.go | 98 ++++--- internal/handler/utils.go | 9 + internal/{ => service}/frostfs/frostfs.go | 4 +- .../service/frostfs/multi_object_reader.go | 241 ++++++++++++++++++ .../frostfs/multi_object_reader_test.go | 137 ++++++++++ .../frostfs}/pool_wrapper.go | 2 +- tree/tree_test.go | 22 +- 11 files changed, 517 insertions(+), 62 deletions(-) rename internal/{ => service}/frostfs/frostfs.go (97%) create mode 100644 internal/service/frostfs/multi_object_reader.go create mode 100644 internal/service/frostfs/multi_object_reader_test.go rename internal/{frostfs/services => service/frostfs}/pool_wrapper.go (99%) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 8d5930d..f8300ec 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -19,11 +19,10 @@ import ( v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" @@ -453,7 +452,7 @@ func (a *app) setHealthStatus() { } func (a *app) Serve() { - handler := handler.New(a.AppParams(), a.settings, tree.NewTree(services.NewPoolWrapper(a.treePool))) + handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool))) // Configure router. a.configureRouter(handler) diff --git a/internal/handler/frostfs_mock.go b/internal/handler/frostfs_mock.go index 9f4378a..b60915e 100644 --- a/internal/handler/frostfs_mock.go +++ b/internal/handler/frostfs_mock.go @@ -229,6 +229,10 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) (Res return &resObjectSearchMock{res: res}, nil } +func (t *TestFrostFS) InitMultiObjectReader(context.Context, PrmInitMultiObjectReader) (io.Reader, error) { + return nil, nil +} + func isMatched(attributes []object.Attribute, filter object.SearchFilter) bool { for _, attr := range attributes { if attr.Key() == filter.Header() { @@ -269,10 +273,3 @@ func (t *TestFrostFS) isAllowed(cnrID cid.ID, userID user.ID, op acl.Op, objID o } return false } - -func newAddress(cnr cid.ID, obj oid.ID) oid.Address { - var addr oid.Address - addr.SetContainer(cnr) - addr.SetObject(obj) - return addr -} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 8b07af0..c680706 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -119,6 +119,14 @@ type PrmObjectSearch struct { Filters object.SearchFilters } +type PrmInitMultiObjectReader struct { + // payload range + Off, Ln uint64 + + Addr oid.Address + Bearer *bearer.Token +} + type ResObjectSearch interface { Read(buf []oid.ID) (int, error) Iterate(f func(oid.ID) bool) error @@ -140,6 +148,8 @@ type FrostFS interface { RangeObject(context.Context, PrmObjectRange) (io.ReadCloser, error) CreateObject(context.Context, PrmObjectCreate) (oid.ID, error) SearchObjects(context.Context, PrmObjectSearch) (ResObjectSearch, error) + InitMultiObjectReader(ctx context.Context, p PrmInitMultiObjectReader) (io.Reader, error) + utils.EpochInfoFetcher } @@ -201,9 +211,7 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ return } - var addr oid.Address - addr.SetContainer(bktInfo.CID) - addr.SetObject(*objID) + addr := newAddress(bktInfo.CID, *objID) f(ctx, *h.newRequest(c, log), addr) } @@ -256,10 +264,7 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r response.Error(c, "object deleted", fasthttp.StatusNotFound) return } - - var addr oid.Address - addr.SetContainer(bktInfo.CID) - addr.SetObject(foundOid.OID) + addr := newAddress(bktInfo.CID, foundOid.OID) f(ctx, *h.newRequest(c, log), addr) } diff --git a/internal/handler/multipart.go b/internal/handler/multipart.go index de9242f..213286c 100644 --- a/internal/handler/multipart.go +++ b/internal/handler/multipart.go @@ -1,13 +1,17 @@ package handler import ( + "errors" "io" + "strconv" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/multipart" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "go.uber.org/zap" ) +const attributeMultipartObjectSize = "S3-Multipart-Object-Size" + // MultipartFile provides standard ReadCloser interface and also allows one to // get file name, it's used for multipart uploads. type MultipartFile interface { @@ -45,3 +49,30 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF return part, nil } } + +// getPayload returns initial payload if object is not multipart else composes new reader with parts data. +func (h *Handler) getPayload(p getMultiobjectBodyParams) (io.ReadCloser, uint64, error) { + cid, ok := p.obj.Header.ContainerID() + if !ok { + return nil, 0, errors.New("no container id set") + } + oid, ok := p.obj.Header.ID() + if !ok { + return nil, 0, errors.New("no object id set") + } + size, err := strconv.ParseUint(p.strSize, 10, 64) + if err != nil { + return nil, 0, err + } + ctx := p.req.RequestCtx + params := PrmInitMultiObjectReader{ + Addr: newAddress(cid, oid), + Bearer: bearerToken(ctx), + } + payload, err := h.frostfs.InitMultiObjectReader(ctx, params) + if err != nil { + return nil, 0, err + } + + return io.NopCloser(payload), size, nil +} diff --git a/internal/handler/reader.go b/internal/handler/reader.go index d901f25..50121c9 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -47,20 +47,26 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (str return http.DetectContentType(buf), buf, err // to not lose io.EOF } -func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oid.Address) { +type getMultiobjectBodyParams struct { + obj *Object + req request + strSize string +} + +func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.Address) { var ( - err error - dis = "inline" - start = time.Now() - filename string - filepath string + shouldDownload = req.QueryArgs().GetBool("download") + start = time.Now() + filename string + filepath string + contentType string ) prm := PrmObjectGet{ PrmAuth: PrmAuth{ BearerToken: bearerToken(ctx), }, - Address: objectAddress, + Address: objAddress, } rObj, err := h.frostfs.GetObject(ctx, prm) @@ -70,15 +76,9 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi } // we can't close reader in this function, so how to do it? - - if req.Request.URI().QueryArgs().GetBool("download") { - dis = "attachment" - } - + req.setIDs(rObj.Header) + payload := rObj.Payload payloadSize := rObj.Header.PayloadSize() - - req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) - var contentType string for _, attr := range rObj.Header.Attributes() { key := attr.Key() val := attr.Value() @@ -93,31 +93,41 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi case object.AttributeFileName: filename = val case object.AttributeTimestamp: - value, err := strconv.ParseInt(val, 10, 64) - if err != nil { - req.log.Info(logs.CouldntParseCreationDate, - zap.String("key", key), + if err = req.setTimestamp(val); err != nil { + req.log.Error(logs.CouldntParseCreationDate, zap.String("val", val), zap.Error(err)) - continue } - req.Response.Header.Set(fasthttp.HeaderLastModified, - time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val case object.AttributeFilePath: filepath = val + case attributeMultipartObjectSize: + payload, payloadSize, err = h.getPayload(getMultiobjectBodyParams{ + obj: rObj, + req: req, + strSize: val, + }) + if err != nil { + req.handleFrostFSErr(err, start) + return + } } } + if filename == "" { + filename = filepath + } - idsToResponse(&req.Response, &rObj.Header) + req.setDisposition(shouldDownload, filename) + + req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) if len(contentType) == 0 { // determine the Content-Type from the payload head var payloadHead []byte contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) { - return rObj.Payload, nil + return payload, nil }) if err != nil && err != io.EOF { req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) @@ -129,20 +139,46 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi var headReader io.Reader = bytes.NewReader(payloadHead) if err != io.EOF { // otherwise, we've already read full payload - headReader = io.MultiReader(headReader, rObj.Payload) + headReader = io.MultiReader(headReader, payload) } // note: we could do with io.Reader, but SetBodyStream below closes body stream // if it implements io.Closer and that's useful for us. - rObj.Payload = readCloser{headReader, rObj.Payload} + payload = readCloser{headReader, payload} } req.SetContentType(contentType) + req.Response.SetBodyStream(payload, int(payloadSize)) +} - if filename == "" { - filename = filepath +func (r *request) setIDs(obj object.Object) { + objID, _ := obj.ID() + cnrID, _ := obj.ContainerID() + r.Response.Header.Set(hdrObjectID, objID.String()) + r.Response.Header.Set(hdrOwnerID, obj.OwnerID().String()) + r.Response.Header.Set(hdrContainerID, cnrID.String()) +} + +func (r *request) setDisposition(shouldDownload bool, filename string) { + const ( + inlineDisposition = "inline" + attachmentDisposition = "attachment" + ) + + dis := inlineDisposition + if shouldDownload { + dis = attachmentDisposition } - req.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) - - req.Response.SetBodyStream(rObj.Payload, int(payloadSize)) + r.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) +} + +func (r *request) setTimestamp(timestamp string) error { + value, err := strconv.ParseInt(timestamp, 10, 64) + if err != nil { + return err + } + r.Response.Header.Set(fasthttp.HeaderLastModified, + time.Unix(value, 0).UTC().Format(http.TimeFormat)) + + return nil } diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 7fa1158..a944b67 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -12,6 +12,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -90,3 +92,10 @@ func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { } response.Error(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) } + +func newAddress(cnr cid.ID, obj oid.ID) oid.Address { + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(obj) + return addr +} diff --git a/internal/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go similarity index 97% rename from internal/frostfs/frostfs.go rename to internal/service/frostfs/frostfs.go index 6cf162a..c7e56a4 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -33,9 +33,9 @@ func NewFrostFS(p *pool.Pool) *FrostFS { } // Container implements frostfs.FrostFS interface method. -func (x *FrostFS) Container(ctx context.Context, layerPrm handler.PrmContainer) (*container.Container, error) { +func (x *FrostFS) Container(ctx context.Context, containerPrm handler.PrmContainer) (*container.Container, error) { prm := pool.PrmContainerGet{ - ContainerID: layerPrm.ContainerID, + ContainerID: containerPrm.ContainerID, } res, err := x.pool.GetContainer(ctx, prm) diff --git a/internal/service/frostfs/multi_object_reader.go b/internal/service/frostfs/multi_object_reader.go new file mode 100644 index 0000000..93f1f60 --- /dev/null +++ b/internal/service/frostfs/multi_object_reader.go @@ -0,0 +1,241 @@ +package frostfs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +// PartInfo is upload information about part. +type PartInfo struct { + Key string `json:"key"` + UploadID string `json:"uploadId"` + Number int `json:"number"` + OID oid.ID `json:"oid"` + Size uint64 `json:"size"` + ETag string `json:"etag"` + MD5 string `json:"md5"` + Created time.Time `json:"created"` +} + +type GetFrostFSParams struct { + // payload range + Off, Ln uint64 + Addr oid.Address +} + +type PartObj struct { + OID oid.ID + Size uint64 +} + +type readerInitiator interface { + InitFrostFSObjectPayloadReader(ctx context.Context, p GetFrostFSParams) (io.ReadCloser, error) +} + +// MultiObjectReader implements io.Reader of payloads of the object list stored in the FrostFS network. +type MultiObjectReader struct { + ctx context.Context + + layer readerInitiator + + startPartOffset uint64 + endPartLength uint64 + + prm GetFrostFSParams + + curIndex int + curReader io.ReadCloser + + parts []PartObj +} + +type MultiObjectReaderConfig struct { + Initiator readerInitiator + + // the offset of complete object and total size to read + Off, Ln uint64 + + Addr oid.Address + Parts []PartObj +} + +var ( + errOffsetIsOutOfRange = errors.New("offset is out of payload range") + errLengthIsOutOfRange = errors.New("length is out of payload range") + errEmptyPartsList = errors.New("empty parts list") + errorZeroRangeLength = errors.New("zero range length") +) + +func (x *FrostFS) InitMultiObjectReader(ctx context.Context, p handler.PrmInitMultiObjectReader) (io.Reader, error) { + combinedObj, err := x.GetObject(ctx, handler.PrmObjectGet{ + PrmAuth: handler.PrmAuth{BearerToken: p.Bearer}, + Address: p.Addr, + }) + if err != nil { + return nil, fmt.Errorf("get combined object '%s': %w", p.Addr.Object().EncodeToString(), err) + } + + var parts []*PartInfo + if err = json.NewDecoder(combinedObj.Payload).Decode(&parts); err != nil { + return nil, fmt.Errorf("unmarshal combined object parts: %w", err) + } + + objParts := make([]PartObj, len(parts)) + for i, part := range parts { + objParts[i] = PartObj{ + OID: part.OID, + Size: part.Size, + } + } + + return NewMultiObjectReader(ctx, MultiObjectReaderConfig{ + Initiator: x, + Off: p.Off, + Ln: p.Ln, + Parts: objParts, + Addr: p.Addr, + }) +} + +func NewMultiObjectReader(ctx context.Context, cfg MultiObjectReaderConfig) (*MultiObjectReader, error) { + if len(cfg.Parts) == 0 { + return nil, errEmptyPartsList + } + + r := &MultiObjectReader{ + ctx: ctx, + layer: cfg.Initiator, + prm: GetFrostFSParams{ + Addr: cfg.Addr, + }, + parts: cfg.Parts, + } + + if cfg.Off+cfg.Ln == 0 { + return r, nil + } + + if cfg.Off > 0 && cfg.Ln == 0 { + return nil, errorZeroRangeLength + } + + startPartIndex, startPartOffset := findStartPart(cfg) + if startPartIndex == -1 { + return nil, errOffsetIsOutOfRange + } + r.startPartOffset = startPartOffset + + endPartIndex, endPartLength := findEndPart(cfg) + if endPartIndex == -1 { + return nil, errLengthIsOutOfRange + } + r.endPartLength = endPartLength + + r.parts = cfg.Parts[startPartIndex : endPartIndex+1] + + return r, nil +} + +func findStartPart(cfg MultiObjectReaderConfig) (index int, offset uint64) { + position := cfg.Off + for i, part := range cfg.Parts { + // Strict inequality when searching for start position to avoid reading zero length part. + if position < part.Size { + return i, position + } + position -= part.Size + } + + return -1, 0 +} + +func findEndPart(cfg MultiObjectReaderConfig) (index int, length uint64) { + position := cfg.Off + cfg.Ln + for i, part := range cfg.Parts { + // Non-strict inequality when searching for end position to avoid out of payload range error. + if position <= part.Size { + return i, position + } + position -= part.Size + } + + return -1, 0 +} + +func (x *MultiObjectReader) Read(p []byte) (n int, err error) { + if x.curReader != nil { + n, err = x.curReader.Read(p) + if err != nil { + if closeErr := x.curReader.Close(); closeErr != nil { + return n, fmt.Errorf("%w (close err: %v)", err, closeErr) + } + } + if !errors.Is(err, io.EOF) { + return n, err + } + + x.curIndex++ + } + + if x.curIndex == len(x.parts) { + return n, io.EOF + } + + x.prm.Addr.SetObject(x.parts[x.curIndex].OID) + + if x.curIndex == 0 { + x.prm.Off = x.startPartOffset + x.prm.Ln = x.parts[x.curIndex].Size - x.startPartOffset + } + + if x.curIndex == len(x.parts)-1 { + x.prm.Ln = x.endPartLength - x.prm.Off + } + + x.curReader, err = x.layer.InitFrostFSObjectPayloadReader(x.ctx, x.prm) + if err != nil { + return n, fmt.Errorf("init payload reader for the next part: %w", err) + } + + x.prm.Off = 0 + x.prm.Ln = 0 + + next, err := x.Read(p[n:]) + + return n + next, err +} + +// InitFrostFSObjectPayloadReader initializes payload reader of the FrostFS object. +// Zero range corresponds to full payload (panics if only offset is set). +func (x *FrostFS) InitFrostFSObjectPayloadReader(ctx context.Context, p GetFrostFSParams) (io.ReadCloser, error) { + var prmAuth handler.PrmAuth + + if p.Off+p.Ln != 0 { + prm := handler.PrmObjectRange{ + PrmAuth: prmAuth, + PayloadRange: [2]uint64{p.Off, p.Ln}, + Address: p.Addr, + } + + return x.RangeObject(ctx, prm) + } + + prm := handler.PrmObjectGet{ + PrmAuth: prmAuth, + Address: p.Addr, + } + + res, err := x.GetObject(ctx, prm) + if err != nil { + return nil, err + } + + return res.Payload, nil +} diff --git a/internal/service/frostfs/multi_object_reader_test.go b/internal/service/frostfs/multi_object_reader_test.go new file mode 100644 index 0000000..4127cdc --- /dev/null +++ b/internal/service/frostfs/multi_object_reader_test.go @@ -0,0 +1,137 @@ +package frostfs + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "testing" + + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +type readerInitiatorMock struct { + parts map[oid.ID][]byte +} + +func (r *readerInitiatorMock) InitFrostFSObjectPayloadReader(_ context.Context, p GetFrostFSParams) (io.ReadCloser, error) { + partPayload, ok := r.parts[p.Addr.Object()] + if !ok { + return nil, errors.New("part not found") + } + + if p.Off+p.Ln == 0 { + return io.NopCloser(bytes.NewReader(partPayload)), nil + } + + if p.Off > uint64(len(partPayload)-1) { + return nil, fmt.Errorf("invalid offset: %d/%d", p.Off, len(partPayload)) + } + + if p.Off+p.Ln > uint64(len(partPayload)) { + return nil, fmt.Errorf("invalid range: %d-%d/%d", p.Off, p.Off+p.Ln, len(partPayload)) + } + + return io.NopCloser(bytes.NewReader(partPayload[p.Off : p.Off+p.Ln])), nil +} + +func prepareDataReader() ([]byte, []PartObj, *readerInitiatorMock) { + mockInitReader := &readerInitiatorMock{ + parts: map[oid.ID][]byte{ + oidtest.ID(): []byte("first part 1"), + oidtest.ID(): []byte("second part 2"), + oidtest.ID(): []byte("third part 3"), + }, + } + + var fullPayload []byte + parts := make([]PartObj, 0, len(mockInitReader.parts)) + for id, payload := range mockInitReader.parts { + parts = append(parts, PartObj{OID: id, Size: uint64(len(payload))}) + fullPayload = append(fullPayload, payload...) + } + + return fullPayload, parts, mockInitReader +} + +func TestMultiReader(t *testing.T) { + ctx := context.Background() + + fullPayload, parts, mockInitReader := prepareDataReader() + + for _, tc := range []struct { + name string + off uint64 + ln uint64 + err error + }{ + { + name: "simple read all", + }, + { + name: "simple read with length", + ln: uint64(len(fullPayload)), + }, + { + name: "middle of parts", + off: parts[0].Size + 2, + ln: 4, + }, + { + name: "first and second", + off: parts[0].Size - 4, + ln: 8, + }, + { + name: "first and third", + off: parts[0].Size - 4, + ln: parts[1].Size + 8, + }, + { + name: "second part", + off: parts[0].Size, + ln: parts[1].Size, + }, + { + name: "second and third", + off: parts[0].Size, + ln: parts[1].Size + parts[2].Size, + }, + { + name: "offset out of range", + off: uint64(len(fullPayload) + 1), + ln: 1, + err: errOffsetIsOutOfRange, + }, + { + name: "zero length", + off: parts[1].Size + 1, + ln: 0, + err: errorZeroRangeLength, + }, + } { + t.Run(tc.name, func(t *testing.T) { + multiReader, err := NewMultiObjectReader(ctx, MultiObjectReaderConfig{ + Initiator: mockInitReader, + Parts: parts, + Off: tc.off, + Ln: tc.ln, + }) + require.ErrorIs(t, err, tc.err) + + if tc.err == nil { + off := tc.off + ln := tc.ln + if off+ln == 0 { + ln = uint64(len(fullPayload)) + } + data, err := io.ReadAll(multiReader) + require.NoError(t, err) + require.Equal(t, fullPayload[off:off+ln], data) + } + }) + } +} diff --git a/internal/frostfs/services/pool_wrapper.go b/internal/service/frostfs/pool_wrapper.go similarity index 99% rename from internal/frostfs/services/pool_wrapper.go rename to internal/service/frostfs/pool_wrapper.go index fa70f15..b978d73 100644 --- a/internal/frostfs/services/pool_wrapper.go +++ b/internal/service/frostfs/pool_wrapper.go @@ -1,4 +1,4 @@ -package services +package frostfs import ( "context" diff --git a/tree/tree_test.go b/tree/tree_test.go index 69ac5f6..62f9914 100644 --- a/tree/tree_test.go +++ b/tree/tree_test.go @@ -21,11 +21,11 @@ func (m nodeMeta) GetValue() []byte { type nodeResponse struct { meta []nodeMeta - timestamp uint64 + timestamp []uint64 } func (n nodeResponse) GetTimestamp() []uint64 { - return []uint64{n.timestamp} + return n.timestamp } func (n nodeResponse) GetMeta() []Meta { @@ -59,7 +59,7 @@ func TestGetLatestNode(t *testing.T) { name: "one node of the object version", nodes: []NodeResponse{ nodeResponse{ - timestamp: 1, + timestamp: []uint64{1}, meta: []nodeMeta{ { key: oidKV, @@ -74,11 +74,11 @@ func TestGetLatestNode(t *testing.T) { name: "one node of the object version and one node of the secondary object", nodes: []NodeResponse{ nodeResponse{ - timestamp: 3, + timestamp: []uint64{3}, meta: []nodeMeta{}, }, nodeResponse{ - timestamp: 1, + timestamp: []uint64{1}, meta: []nodeMeta{ { key: oidKV, @@ -93,11 +93,11 @@ func TestGetLatestNode(t *testing.T) { name: "all nodes represent a secondary object", nodes: []NodeResponse{ nodeResponse{ - timestamp: 3, + timestamp: []uint64{3}, meta: []nodeMeta{}, }, nodeResponse{ - timestamp: 5, + timestamp: []uint64{5}, meta: []nodeMeta{}, }, }, @@ -107,7 +107,7 @@ func TestGetLatestNode(t *testing.T) { name: "several nodes of different types and with different timestamp", nodes: []NodeResponse{ nodeResponse{ - timestamp: 1, + timestamp: []uint64{1}, meta: []nodeMeta{ { key: oidKV, @@ -116,11 +116,11 @@ func TestGetLatestNode(t *testing.T) { }, }, nodeResponse{ - timestamp: 3, + timestamp: []uint64{3}, meta: []nodeMeta{}, }, nodeResponse{ - timestamp: 4, + timestamp: []uint64{4}, meta: []nodeMeta{ { key: oidKV, @@ -129,7 +129,7 @@ func TestGetLatestNode(t *testing.T) { }, }, nodeResponse{ - timestamp: 6, + timestamp: []uint64{6}, meta: []nodeMeta{}, }, }, From fc86ab3511a153b30b7ff45b15ff5865405786ca Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Tue, 8 Oct 2024 12:08:39 +0300 Subject: [PATCH 12/86] [#148] Add trace_id to logs Signed-off-by: Roman Loginov --- CHANGELOG.md | 3 +++ cmd/http-gw/app.go | 41 ++++++++++++++++++++++++------------ internal/handler/browse.go | 4 +++- internal/handler/download.go | 9 ++++---- internal/handler/handler.go | 35 +++++++++++++++--------------- internal/handler/upload.go | 28 +++++++++++++----------- utils/util.go | 32 ++++++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a489027..46e9c23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,10 @@ This document outlines major changes between releases. ## [Unreleased] + +### Added - Support percent-encoding for GET queries (#134) +- Add `trace_id` to logs (#148) ### Changed - Update go version to 1.22 (#132) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index f8300ec..b26df89 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -42,6 +42,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/viper" "github.com/valyala/fasthttp" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "golang.org/x/exp/slices" ) @@ -589,15 +590,15 @@ func (a *app) configureRouter(handler *handler.Handler) { response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.Upload)))))) + r.POST("/upload/{cid}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.Upload)))))) a.log.Info(logs.AddedPathUploadCid) - r.GET("/get/{cid}/{oid:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadByAddressOrBucketName)))))) - r.HEAD("/get/{cid}/{oid:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.HeadByAddressOrBucketName)))))) + r.GET("/get/{cid}/{oid:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadByAddressOrBucketName)))))) + r.HEAD("/get/{cid}/{oid:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.HeadByAddressOrBucketName)))))) a.log.Info(logs.AddedPathGetCidOid) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadByAttribute)))))) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.HeadByAttribute)))))) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadByAttribute)))))) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.HeadByAttribute)))))) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadZipped)))))) + r.GET("/zip/{cid}/{prefix:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadZipped)))))) a.log.Info(logs.AddedPathZipCidPrefix) a.webServer.Handler = r.Handler @@ -605,11 +606,24 @@ func (a *app) configureRouter(handler *handler.Handler) { func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { - a.log.Info(logs.Request, zap.String("remote", req.RemoteAddr().String()), + requiredFields := []zap.Field{zap.Uint64("id", req.ID())} + reqCtx := utils.GetContextFromRequest(req) + if traceID := trace.SpanFromContext(reqCtx).SpanContext().TraceID(); traceID.IsValid() { + requiredFields = append(requiredFields, zap.String("trace_id", traceID.String())) + } + log := a.log.With(requiredFields...) + + reqCtx = utils.SetReqLog(reqCtx, log) + utils.SetContextToRequest(reqCtx, req) + + fields := []zap.Field{ + zap.String("remote", req.RemoteAddr().String()), zap.ByteString("method", req.Method()), zap.ByteString("path", req.Path()), zap.ByteString("query", req.QueryArgs().QueryString()), - zap.Uint64("id", req.ID())) + } + + log.Info(logs.Request, fields...) h(req) } } @@ -648,9 +662,12 @@ func (a *app) canonicalizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { - appCtx, err := tokens.StoreBearerTokenAppCtx(a.ctx, req) + reqCtx := utils.GetContextFromRequest(req) + appCtx, err := tokens.StoreBearerTokenAppCtx(reqCtx, req) if err != nil { - a.log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Uint64("id", req.ID()), zap.Error(err)) + log := utils.GetReqLogOrDefault(reqCtx, a.log) + + log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err)) response.Error(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -661,9 +678,7 @@ func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { func (a *app) tracer(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { - appCtx := utils.GetContextFromRequest(req) - - appCtx, span := utils.StartHTTPServerSpan(appCtx, req, "REQUEST") + appCtx, span := utils.StartHTTPServerSpan(a.ctx, req, "REQUEST") defer func() { utils.SetHTTPTraceInfo(appCtx, span, req) span.End() diff --git a/internal/handler/browse.go b/internal/handler/browse.go index e84fb04..c89d8c5 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -113,8 +113,10 @@ func urlencode(prefix, filename string) string { } func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) { - log := h.log.With(zap.String("bucket", bucketInfo.Name)) ctx := utils.GetContextFromRequest(c) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("bucket", bucketInfo.Name)) + nodes, err := h.listObjects(ctx, bucketInfo, prefix) if err != nil { logAndSendBucketError(c, log, err) diff --git a/internal/handler/download.go b/internal/handler/download.go index 88109a6..19380d4 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -84,16 +84,17 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { scid, _ := c.UserValue("cid").(string) prefix, _ := c.UserValue("prefix").(string) + ctx := utils.GetContextFromRequest(c) + log := utils.GetReqLogOrDefault(ctx, h.log) + prefix, err := url.QueryUnescape(prefix) if err != nil { - h.log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Uint64("id", c.ID()), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) response.Error(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) return } - log := h.log.With(zap.String("cid", scid), zap.String("prefix", prefix), zap.Uint64("id", c.ID())) - - ctx := utils.GetContextFromRequest(c) + log = log.With(zap.String("cid", scid), zap.String("prefix", prefix)) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index c680706..62d0897 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -190,13 +190,12 @@ func New(params *AppParams, config Config, tree *tree.Tree) *Handler { // byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - var ( - idCnr, _ = c.UserValue("cid").(string) - idObj, _ = c.UserValue("oid").(string) - log = h.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) - ) + idCnr, _ := c.UserValue("cid").(string) + idObj, _ := c.UserValue("oid").(string) ctx := utils.GetContextFromRequest(c) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("cid", idCnr), zap.String("oid", idObj)) bktInfo, err := h.getBucketInfo(ctx, idCnr, log) if err != nil { @@ -219,12 +218,13 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ // byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - var ( - bucketname = c.UserValue("cid").(string) - key = c.UserValue("oid").(string) - log = h.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) - download = c.QueryArgs().GetBool("download") - ) + bucketname := c.UserValue("cid").(string) + key := c.UserValue("oid").(string) + download := c.QueryArgs().GetBool("download") + + ctx := utils.GetContextFromRequest(c) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("bucketname", bucketname), zap.String("key", key)) unescapedKey, err := url.QueryUnescape(key) if err != nil { @@ -232,8 +232,6 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r return } - ctx := utils.GetContextFromRequest(c) - bktInfo, err := h.getBucketInfo(ctx, bucketname, log) if err != nil { logAndSendBucketError(c, log, err) @@ -275,23 +273,24 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re key, _ := c.UserValue("attr_key").(string) val, _ := c.UserValue("attr_val").(string) + ctx := utils.GetContextFromRequest(c) + log := utils.GetReqLogOrDefault(ctx, h.log) + key, err := url.QueryUnescape(key) if err != nil { - h.log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_key", key), zap.Uint64("id", c.ID()), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_key", key), zap.Error(err)) response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) return } val, err = url.QueryUnescape(val) if err != nil { - h.log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_val", val), zap.Uint64("id", c.ID()), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_val", val), zap.Error(err)) response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) return } - log := h.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) - - ctx := utils.GetContextFromRequest(c) + log = log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 6c0e117..867025d 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -45,16 +45,18 @@ func (pr *putResponse) encode(w io.Writer) error { // Upload handles multipart upload request. func (h *Handler) Upload(c *fasthttp.RequestCtx) { var ( - file MultipartFile - idObj oid.ID - addr oid.Address - scid, _ = c.UserValue("cid").(string) - log = h.log.With(zap.String("cid", scid)) - bodyStream = c.RequestBodyStream() - drainBuf = make([]byte, drainBufSize) + file MultipartFile + idObj oid.ID + addr oid.Address ) + scid, _ := c.UserValue("cid").(string) + bodyStream := c.RequestBodyStream() + drainBuf := make([]byte, drainBufSize) + ctx := utils.GetContextFromRequest(c) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("cid", scid)) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { @@ -75,13 +77,15 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { zap.Error(err), ) }() + boundary := string(c.Request.Header.MultipartFormBoundary()) - if file, err = fetchMultipartFile(h.log, bodyStream, boundary); err != nil { + if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } - filtered, err := filterHeaders(h.log, &c.Request.Header) + + filtered, err := filterHeaders(log, &c.Request.Header) if err != nil { log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) response.Error(c, err.Error(), fasthttp.StatusBadRequest) @@ -143,7 +147,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { } if idObj, err = h.frostfs.CreateObject(ctx, prm); err != nil { - h.handlePutFrostFSErr(c, err) + h.handlePutFrostFSErr(c, err, log) return } @@ -174,11 +178,11 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { c.Response.Header.SetContentType(jsonHeader) } -func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) { +func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *zap.Logger) { statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err) logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) - h.log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) + log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) response.Error(r, msg, statusCode) } diff --git a/utils/util.go b/utils/util.go index d513817..b7f5e39 100644 --- a/utils/util.go +++ b/utils/util.go @@ -4,6 +4,7 @@ import ( "context" "github.com/valyala/fasthttp" + "go.uber.org/zap" ) // SetContextToRequest adds new context to fasthttp request. @@ -15,3 +16,34 @@ func SetContextToRequest(ctx context.Context, c *fasthttp.RequestCtx) { func GetContextFromRequest(c *fasthttp.RequestCtx) context.Context { return c.UserValue("context").(context.Context) } + +type ctxReqLoggerKeyType struct{} + +// SetReqLog sets child zap.Logger in the context. +func SetReqLog(ctx context.Context, log *zap.Logger) context.Context { + if ctx == nil { + return nil + } + return context.WithValue(ctx, ctxReqLoggerKeyType{}, log) +} + +// GetReqLog returns log if set. +// If zap.Logger isn't set returns nil. +func GetReqLog(ctx context.Context) *zap.Logger { + if ctx == nil { + return nil + } else if r, ok := ctx.Value(ctxReqLoggerKeyType{}).(*zap.Logger); ok { + return r + } + return nil +} + +// GetReqLogOrDefault returns log from context, if it exists. +// If the log is missing from the context, the default logger is returned. +func GetReqLogOrDefault(ctx context.Context, defaultLog *zap.Logger) *zap.Logger { + log := GetReqLog(ctx) + if log == nil { + log = defaultLog + } + return log +} From 70846fdaecf6ee2867507ac8e75b82ac61cc35db Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Mon, 21 Oct 2024 11:39:14 +0300 Subject: [PATCH 13/86] [#157] Support the continuous use of interceptors We can always add interceptors to the grpc connection to the storage, since the actual use will be controlled by the configuration from the frostfs-observability library. Signed-off-by: Roman Loginov --- cmd/http-gw/settings.go | 16 +++++----------- docs/gate-configuration.md | 2 +- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index eab5b6b..476d658 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -589,18 +589,12 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool. prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgTreePoolMaxAttempts)) - var apiGRPCDialOpts []grpc.DialOption - var treeGRPCDialOpts []grpc.DialOption - if cfg.GetBool(cfgTracingEnabled) { - interceptors := []grpc.DialOption{ - grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), - grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), - } - treeGRPCDialOpts = append(treeGRPCDialOpts, interceptors...) - apiGRPCDialOpts = append(apiGRPCDialOpts, interceptors...) + interceptors := []grpc.DialOption{ + grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), + grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), } - prm.SetGRPCDialOptions(apiGRPCDialOpts...) - prmTree.SetGRPCDialOptions(treeGRPCDialOpts...) + prm.SetGRPCDialOptions(interceptors...) + prmTree.SetGRPCDialOptions(interceptors...) p, err := pool.NewPool(prm) if err != nil { diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index e8d1f4b..b484f9d 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -271,7 +271,7 @@ tracing: | Parameter | Type | SIGHUP reload | Default value | Description | |--------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| -| `enabled` | `bool` | no | `false` | Flag to enable the tracing. | +| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | | `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | | `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | | `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | From 8dc527296515bd255ab8d6920b21a1e2133eea98 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 24 Oct 2024 16:17:28 +0300 Subject: [PATCH 14/86] [#158] Rework app settings Update settings by sighup using one lock/unlock operation Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 150 ++++++++++++---------------------------- cmd/http-gw/settings.go | 32 +++++++++ 2 files changed, 76 insertions(+), 106 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index b26df89..ead4a29 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -6,13 +6,11 @@ import ( "crypto/x509" "errors" "fmt" - "io" "net/http" "os" "os/signal" "runtime/debug" "strconv" - "strings" "sync" "syscall" "time" @@ -159,23 +157,47 @@ func newApp(ctx context.Context, opt ...Option) App { a.initResolver() a.initMetrics() a.initTracing(ctx) - a.loadIndexPageTemplate() return a } +func (a *app) initAppSettings() { + a.settings = &appSettings{ + reconnectInterval: fetchReconnectInterval(a.cfg), + } + a.settings.update(a.cfg, a.log) +} + +func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { + defaultTimestamp := v.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) + zipCompression := v.GetBool(cfgZipCompression) + returnIndexPage := v.GetBool(cfgIndexPageEnabled) + clientCut := v.GetBool(cfgClientCut) + bufferMaxSizeForPut := v.GetUint64(cfgBufferMaxSizeForPut) + namespaceHeader := v.GetString(cfgResolveNamespaceHeader) + defaultNamespaces := fetchDefaultNamespaces(v) + indexPage, indexEnabled := fetchIndexPageTemplate(v, l) + + s.mu.Lock() + defer s.mu.Unlock() + + s.defaultTimestamp = defaultTimestamp + s.zipCompression = zipCompression + s.returnIndexPage = returnIndexPage + s.clientCut = clientCut + s.bufferMaxSizeForPut = bufferMaxSizeForPut + s.namespaceHeader = namespaceHeader + s.defaultNamespaces = defaultNamespaces + s.returnIndexPage = indexEnabled + s.indexPageTemplate = indexPage +} + func (s *appSettings) DefaultTimestamp() bool { s.mu.RLock() defer s.mu.RUnlock() return s.defaultTimestamp } -func (s *appSettings) setDefaultTimestamp(val bool) { - s.mu.Lock() - s.defaultTimestamp = val - s.mu.Unlock() -} - func (s *appSettings) ZipCompression() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -197,73 +219,33 @@ func (s *appSettings) IndexPageTemplate() string { return s.indexPageTemplate } -func (s *appSettings) setZipCompression(val bool) { - s.mu.Lock() - s.zipCompression = val - s.mu.Unlock() -} - -func (s *appSettings) setReturnIndexPage(val bool) { - s.mu.Lock() - s.returnIndexPage = val - s.mu.Unlock() -} - -func (s *appSettings) setIndexTemplate(val string) { - s.mu.Lock() - s.indexPageTemplate = val - s.mu.Unlock() -} - -func (a *app) loadIndexPageTemplate() { - if !a.settings.IndexPageEnabled() { - return - } - reader, err := os.Open(a.cfg.GetString(cfgIndexPageTemplatePath)) - if err != nil { - a.settings.setIndexTemplate("") - a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) - return - } - tmpl, err := io.ReadAll(reader) - if err != nil { - a.settings.setIndexTemplate("") - a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) - return - } - a.settings.setIndexTemplate(string(tmpl)) - a.log.Info(logs.SetCustomIndexPageTemplate) -} - func (s *appSettings) ClientCut() bool { s.mu.RLock() defer s.mu.RUnlock() return s.clientCut } -func (s *appSettings) setClientCut(val bool) { - s.mu.Lock() - s.clientCut = val - s.mu.Unlock() -} - func (s *appSettings) BufferMaxSizeForPut() uint64 { s.mu.RLock() defer s.mu.RUnlock() return s.bufferMaxSizeForPut } -func (s *appSettings) setBufferMaxSizeForPut(val uint64) { - s.mu.Lock() - s.bufferMaxSizeForPut = val - s.mu.Unlock() +func (s *appSettings) NamespaceHeader() string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.namespaceHeader } -func (a *app) initAppSettings() { - a.settings = &appSettings{ - reconnectInterval: fetchReconnectInterval(a.cfg), +func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) { + s.mu.RLock() + namespaces := s.defaultNamespaces + s.mu.RUnlock() + if slices.Contains(namespaces, ns) { + return v2container.SysAttributeZoneDefault, true } - a.updateSettings() + + return ns + ".ns", false } func (a *app) initResolver() { @@ -539,26 +521,15 @@ func (a *app) configReload(ctx context.Context) { a.stopServices() a.startServices() - a.updateSettings() + a.settings.update(a.cfg, a.log) a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) a.initTracing(ctx) - a.loadIndexPageTemplate() a.setHealthStatus() a.log.Info(logs.SIGHUPConfigReloadCompleted) } -func (a *app) updateSettings() { - a.settings.setDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) - a.settings.setZipCompression(a.cfg.GetBool(cfgZipCompression)) - a.settings.setReturnIndexPage(a.cfg.GetBool(cfgIndexPageEnabled)) - a.settings.setClientCut(a.cfg.GetBool(cfgClientCut)) - a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut)) - a.settings.setNamespaceHeader(a.cfg.GetString(cfgResolveNamespaceHeader)) - a.settings.setDefaultNamespaces(a.cfg.GetStringSlice(cfgResolveDefaultNamespaces)) -} - func (a *app) startServices() { pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)} pprofService := metrics.NewPprofService(a.log, pprofConfig) @@ -847,39 +818,6 @@ func (a *app) setRuntimeParameters() { } } -func (s *appSettings) NamespaceHeader() string { - s.mu.RLock() - defer s.mu.RUnlock() - return s.namespaceHeader -} - -func (s *appSettings) setNamespaceHeader(nsHeader string) { - s.mu.Lock() - s.namespaceHeader = nsHeader - s.mu.Unlock() -} - -func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) { - s.mu.RLock() - namespaces := s.defaultNamespaces - s.mu.RUnlock() - if slices.Contains(namespaces, ns) { - return v2container.SysAttributeZoneDefault, true - } - - return ns + ".ns", false -} - -func (s *appSettings) setDefaultNamespaces(namespaces []string) { - for i := range namespaces { // to be set namespaces in env variable as `HTTP_GW_RESOLVE_BUCKET_DEFAULT_NAMESPACES="" "root"` - namespaces[i] = strings.Trim(namespaces[i], "\"") - } - - s.mu.Lock() - s.defaultNamespaces = namespaces - s.mu.Unlock() -} - func (a *app) scheduleReconnect(ctx context.Context, srv *fasthttp.Server) { go func() { t := time.NewTicker(a.settings.reconnectInterval) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 476d658..e777f67 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "io" "math" "os" "path" @@ -505,6 +506,37 @@ func fetchReconnectInterval(cfg *viper.Viper) time.Duration { return reconnect } +func fetchIndexPageTemplate(v *viper.Viper, l *zap.Logger) (string, bool) { + if !v.GetBool(cfgIndexPageEnabled) { + return "", false + } + + reader, err := os.Open(v.GetString(cfgIndexPageTemplatePath)) + if err != nil { + l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + return "", true + } + + tmpl, err := io.ReadAll(reader) + if err != nil { + l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + return "", true + } + + l.Info(logs.SetCustomIndexPageTemplate) + return string(tmpl), true +} + +func fetchDefaultNamespaces(v *viper.Viper) []string { + namespaces := v.GetStringSlice(cfgResolveDefaultNamespaces) + + for i := range namespaces { // to be set namespaces in env variable as `HTTP_GW_RESOLVE_BUCKET_DEFAULT_NAMESPACES="" "root"` + namespaces[i] = strings.Trim(namespaces[i], "\"") + } + + return namespaces +} + func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { var servers []ServerInfo seen := make(map[string]struct{}) From 901b8ff95b6ca8eb96880dc663b86014d8915d56 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 24 Oct 2024 16:18:24 +0300 Subject: [PATCH 15/86] [#158] Fix integration test compilation error Signed-off-by: Denis Kirillov --- cmd/http-gw/integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index e888ed6..79a2da5 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -102,7 +102,7 @@ func runServer(pathToWallet string) (App, context.CancelFunc) { v.Set(cfgWalletPath, pathToWallet) v.Set(cfgWalletPassphrase, "") - l, lvl := newStdoutLogger(zapcore.DebugLevel) + l, lvl := newStdoutLogger(v, zapcore.DebugLevel) application := newApp(cancelCtx, WithConfig(v), WithLogger(l, lvl)) go application.Serve() @@ -525,7 +525,7 @@ func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID id, err := clientPool.PutObject(ctx, prm) require.NoError(t, err) - return id + return id.ObjectID } func makeBearerToken(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) string { From 46c63edd67af9f1676283db5d89e6b72f7bd32d4 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 24 Oct 2024 17:27:32 +0300 Subject: [PATCH 16/86] [#158] Support cors Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + cmd/http-gw/app.go | 158 +++++++++++++++++++++++++++++++++---- cmd/http-gw/settings.go | 19 +++++ config/config.env | 7 ++ config/config.yaml | 8 ++ docs/gate-configuration.md | 24 ++++++ 6 files changed, 201 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46e9c23..0990b51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This document outlines major changes between releases. ### Added - Support percent-encoding for GET queries (#134) - Add `trace_id` to logs (#148) +- Add `cors` config params (#158) ### Changed - Update go version to 1.22 (#132) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index ead4a29..fa93a0e 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -11,6 +11,7 @@ import ( "os/signal" "runtime/debug" "strconv" + "strings" "sync" "syscall" "time" @@ -87,15 +88,30 @@ type ( appSettings struct { reconnectInterval time.Duration - mu sync.RWMutex - defaultTimestamp bool - zipCompression bool - clientCut bool - returnIndexPage bool - indexPageTemplate string - bufferMaxSizeForPut uint64 - namespaceHeader string - defaultNamespaces []string + mu sync.RWMutex + defaultTimestamp bool + zipCompression bool + clientCut bool + returnIndexPage bool + indexPageTemplate string + bufferMaxSizeForPut uint64 + namespaceHeader string + defaultNamespaces []string + corsAllowOrigin string + corsAllowMethods []string + corsAllowHeaders []string + corsExposeHeaders []string + corsAllowCredentials bool + corsMaxAge int + } + + CORS struct { + AllowOrigin string + AllowMethods []string + AllowHeaders []string + ExposeHeaders []string + AllowCredentials bool + MaxAge int } ) @@ -177,6 +193,12 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { namespaceHeader := v.GetString(cfgResolveNamespaceHeader) defaultNamespaces := fetchDefaultNamespaces(v) indexPage, indexEnabled := fetchIndexPageTemplate(v, l) + corsAllowOrigin := v.GetString(cfgCORSAllowOrigin) + corsAllowMethods := v.GetStringSlice(cfgCORSAllowMethods) + corsAllowHeaders := v.GetStringSlice(cfgCORSAllowHeaders) + corsExposeHeaders := v.GetStringSlice(cfgCORSExposeHeaders) + corsAllowCredentials := v.GetBool(cfgCORSAllowCredentials) + corsMaxAge := fetchCORSMaxAge(v) s.mu.Lock() defer s.mu.Unlock() @@ -190,6 +212,12 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.defaultNamespaces = defaultNamespaces s.returnIndexPage = indexEnabled s.indexPageTemplate = indexPage + s.corsAllowOrigin = corsAllowOrigin + s.corsAllowMethods = corsAllowMethods + s.corsAllowHeaders = corsAllowHeaders + s.corsExposeHeaders = corsExposeHeaders + s.corsAllowCredentials = corsAllowCredentials + s.corsMaxAge = corsMaxAge } func (s *appSettings) DefaultTimestamp() bool { @@ -219,6 +247,29 @@ func (s *appSettings) IndexPageTemplate() string { return s.indexPageTemplate } +func (s *appSettings) CORS() CORS { + s.mu.RLock() + defer s.mu.RUnlock() + + allowMethods := make([]string, len(s.corsAllowMethods)) + copy(allowMethods, s.corsAllowMethods) + + allowHeaders := make([]string, len(s.corsAllowHeaders)) + copy(allowHeaders, s.corsAllowHeaders) + + exposeHeaders := make([]string, len(s.corsExposeHeaders)) + copy(exposeHeaders, s.corsExposeHeaders) + + return CORS{ + AllowOrigin: s.corsAllowOrigin, + AllowMethods: allowMethods, + AllowHeaders: allowHeaders, + ExposeHeaders: exposeHeaders, + AllowCredentials: s.corsAllowCredentials, + MaxAge: s.corsMaxAge, + } +} + func (s *appSettings) ClientCut() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -550,7 +601,6 @@ func (a *app) stopServices() { svc.ShutDown(ctx) } } - func (a *app) configureRouter(handler *handler.Handler) { r := router.New() r.RedirectTrailingSlash = true @@ -561,20 +611,96 @@ func (a *app) configureRouter(handler *handler.Handler) { response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.Upload)))))) + r.POST("/upload/{cid}", a.addMiddlewares(handler.Upload)) + r.OPTIONS("/upload/{cid}", a.addPreflight()) a.log.Info(logs.AddedPathUploadCid) - r.GET("/get/{cid}/{oid:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadByAddressOrBucketName)))))) - r.HEAD("/get/{cid}/{oid:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.HeadByAddressOrBucketName)))))) + r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(handler.DownloadByAddressOrBucketName)) + r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(handler.HeadByAddressOrBucketName)) + r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetCidOid) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadByAttribute)))))) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.HeadByAttribute)))))) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(handler.DownloadByAttribute)) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(handler.HeadByAttribute)) + r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadZipped)))))) + r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(handler.DownloadZipped)) + r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) a.log.Info(logs.AddedPathZipCidPrefix) a.webServer.Handler = r.Handler } +func (a *app) addMiddlewares(h fasthttp.RequestHandler) fasthttp.RequestHandler { + list := []func(fasthttp.RequestHandler) fasthttp.RequestHandler{ + a.tracer, + a.logger, + a.canonicalizer, + a.tokenizer, + a.reqNamespace, + a.cors, + } + + for i := len(list) - 1; i >= 0; i-- { + h = list[i](h) + } + + return h +} + +func (a *app) addPreflight() fasthttp.RequestHandler { + list := []func(fasthttp.RequestHandler) fasthttp.RequestHandler{ + a.tracer, + a.logger, + a.reqNamespace, + } + + h := a.preflightHandler + for i := len(list) - 1; i >= 0; i-- { + h = list[i](h) + } + + return h +} + +func (a *app) preflightHandler(c *fasthttp.RequestCtx) { + cors := a.settings.CORS() + setCORSHeaders(c, cors) +} + +func (a *app) cors(h fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(c *fasthttp.RequestCtx) { + h(c) + code := c.Response.StatusCode() + if code >= fasthttp.StatusOK && code < fasthttp.StatusMultipleChoices { + cors := a.settings.CORS() + setCORSHeaders(c, cors) + } + } +} + +func setCORSHeaders(c *fasthttp.RequestCtx, cors CORS) { + c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(cors.MaxAge)) + + if len(cors.AllowOrigin) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, cors.AllowOrigin) + } + + if len(cors.AllowMethods) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(cors.AllowMethods, ",")) + } + + if len(cors.AllowHeaders) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, strings.Join(cors.AllowHeaders, ",")) + } + + if len(cors.ExposeHeaders) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(cors.ExposeHeaders, ",")) + } + + if cors.AllowCredentials { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + } +} + func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { requiredFields := []zap.Field{zap.Uint64("id", req.ID())} diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index e777f67..5fb628d 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -56,6 +56,8 @@ const ( defaultReconnectInterval = time.Minute + defaultCORSMaxAge = 600 // seconds + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -141,6 +143,14 @@ const ( cfgResolveNamespaceHeader = "resolve_bucket.namespace_header" cfgResolveDefaultNamespaces = "resolve_bucket.default_namespaces" + // CORS. + cfgCORSAllowOrigin = "cors.allow_origin" + cfgCORSAllowMethods = "cors.allow_methods" + cfgCORSAllowHeaders = "cors.allow_headers" + cfgCORSExposeHeaders = "cors.expose_headers" + cfgCORSAllowCredentials = "cors.allow_credentials" + cfgCORSMaxAge = "cors.max_age" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -537,6 +547,15 @@ func fetchDefaultNamespaces(v *viper.Viper) []string { return namespaces } +func fetchCORSMaxAge(v *viper.Viper) int { + maxAge := v.GetInt(cfgCORSMaxAge) + if maxAge <= 0 { + maxAge = defaultCORSMaxAge + } + + return maxAge +} + func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { var servers []ServerInfo seen := make(map[string]struct{}) diff --git a/config/config.env b/config/config.env index e1d5e7d..d2f4a56 100644 --- a/config/config.env +++ b/config/config.env @@ -126,3 +126,10 @@ HTTP_GW_RESOLVE_BUCKET_DEFAULT_NAMESPACES="" "root" # Max attempt to make successful tree request. # default value is 0 that means the number of attempts equals to number of nodes in pool. HTTP_GW_FROSTFS_TREE_POOL_MAX_ATTEMPTS=0 + +HTTP_GW_CORS_ALLOW_ORIGIN="*" +HTTP_GW_CORS_ALLOW_METHODS="GET" "POST" +HTTP_GW_CORS_ALLOW_HEADERS="*" +HTTP_GW_CORS_EXPOSE_HEADERS="*" +HTTP_GW_CORS_ALLOW_CREDENTIALS=false +HTTP_GW_CORS_MAX_AGE=600 diff --git a/config/config.yaml b/config/config.yaml index 61aa70b..dd985ad 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -138,3 +138,11 @@ cache: resolve_bucket: namespace_header: X-Frostfs-Namespace default_namespaces: [ "", "root" ] + +cors: + allow_origin: "" + allow_methods: [] + allow_headers: [] + expose_headers: [] + allow_credentials: false + max_age: 600 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index b484f9d..7a3eba7 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -363,3 +363,27 @@ index_page: |-----------------|----------|---------------|---------------|---------------------------------------------------------------------------------| | `enabled` | `bool` | yes | `false` | Flag to enable index_page return if no object with specified S3-name was found. | | `template_path` | `string` | yes | `""` | Path to .gotmpl file with html template for index_page. | + +# `cors` section + +Parameters for CORS (used in OPTIONS requests and responses in all handlers). +If values are not set, headers will not be included to response. + +```yaml +cors: + allow_origin: "*" + allow_methods: ["GET", "HEAD"] + allow_headers: ["Authorization"] + expose_headers: ["*"] + allow_credentials: false + max_age: 600 +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------------|------------|---------------|---------------|--------------------------------------------------------| +| `allow_origin` | `string` | yes | | Values for `Access-Control-Allow-Origin` headers. | +| `allow_methods` | `[]string` | yes | | Values for `Access-Control-Allow-Methods` headers. | +| `allow_headers` | `[]string` | yes | | Values for `Access-Control-Allow-Headers` headers. | +| `expose_headers` | `[]string` | yes | | Values for `Access-Control-Expose-Headers` headers. | +| `allow_credentials` | `bool` | yes | `false` | Values for `Access-Control-Allow-Credentials` headers. | +| `max_age` | `int` | yes | `600` | Values for `Access-Control-Max-Age ` headers. | From 69b7761bd6e03cad3f70bffe592716ac8bb52489 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 29 Oct 2024 17:45:42 +0300 Subject: [PATCH 17/86] [#160] Add internal/net package with multinet dialer source Signed-off-by: Alex Vanin --- go.mod | 3 +- go.sum | 2 + internal/logs/logs.go | 2 + internal/net/config.go | 68 ++++++++++++++++++++++++++++++++++ internal/net/dial_target.go | 54 +++++++++++++++++++++++++++ internal/net/dialer.go | 36 ++++++++++++++++++ internal/net/dialer_source.go | 69 +++++++++++++++++++++++++++++++++++ internal/net/event_handler.go | 28 ++++++++++++++ 8 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 internal/net/config.go create mode 100644 internal/net/dial_target.go create mode 100644 internal/net/dialer.go create mode 100644 internal/net/dialer_source.go create mode 100644 internal/net/event_handler.go diff --git a/go.mod b/go.mod index d1a3788..efb79eb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 + git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 github.com/docker/go-units v0.4.0 @@ -25,6 +26,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/net v0.26.0 + golang.org/x/sys v0.22.0 google.golang.org/grpc v1.66.2 ) @@ -106,7 +108,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index bc433eb..ce02ee6 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= +git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= +git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972/go.mod h1:2hM42MBrlhvN6XToaW6OWNk5ZLcu1FhaukGgxtfpDDI= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 96bdaa5..7b7ddc1 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -79,4 +79,6 @@ const ( ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" WarnDuplicateAddress = "duplicate address" + MultinetDialSuccess = "multinet dial successful" + MultinetDialFail = "multinet dial failed" ) diff --git a/internal/net/config.go b/internal/net/config.go new file mode 100644 index 0000000..b40e003 --- /dev/null +++ b/internal/net/config.go @@ -0,0 +1,68 @@ +package net + +import ( + "errors" + "fmt" + "net/netip" + "slices" + "time" + + "git.frostfs.info/TrueCloudLab/multinet" +) + +var errEmptySourceIPList = errors.New("empty source IP list") + +type Subnet struct { + Prefix string + SourceIPs []string +} + +type Config struct { + Enabled bool + Subnets []Subnet + Balancer string + Restrict bool + FallbackDelay time.Duration + EventHandler multinet.EventHandler +} + +func (c Config) toMultinetConfig() (multinet.Config, error) { + var subnets []multinet.Subnet + for _, s := range c.Subnets { + var ms multinet.Subnet + p, err := netip.ParsePrefix(s.Prefix) + if err != nil { + return multinet.Config{}, fmt.Errorf("parse IP prefix '%s': %w", s.Prefix, err) + } + ms.Prefix = p + for _, ip := range s.SourceIPs { + addr, err := netip.ParseAddr(ip) + if err != nil { + return multinet.Config{}, fmt.Errorf("parse IP address '%s': %w", ip, err) + } + ms.SourceIPs = append(ms.SourceIPs, addr) + } + if len(ms.SourceIPs) == 0 { + return multinet.Config{}, errEmptySourceIPList + } + subnets = append(subnets, ms) + } + return multinet.Config{ + Subnets: subnets, + Balancer: multinet.BalancerType(c.Balancer), + Restrict: c.Restrict, + FallbackDelay: c.FallbackDelay, + Dialer: newDefaultDialer(), + EventHandler: c.EventHandler, + }, nil +} + +func (c Config) equals(other Config) bool { + return c.Enabled == other.Enabled && + slices.EqualFunc(c.Subnets, other.Subnets, func(lhs, rhs Subnet) bool { + return lhs.Prefix == rhs.Prefix && slices.Equal(lhs.SourceIPs, rhs.SourceIPs) + }) && + c.Balancer == other.Balancer && + c.Restrict == other.Restrict && + c.FallbackDelay == other.FallbackDelay +} diff --git a/internal/net/dial_target.go b/internal/net/dial_target.go new file mode 100644 index 0000000..6265f18 --- /dev/null +++ b/internal/net/dial_target.go @@ -0,0 +1,54 @@ +// NOTE: code is taken from https://github.com/grpc/grpc-go/blob/v1.68.x/internal/transport/http_util.go + +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package net + +import ( + "net/url" + "strings" +) + +// parseDialTarget returns the network and address to pass to dialer. +func parseDialTarget(target string) (string, string) { + net := "tcp" + m1 := strings.Index(target, ":") + m2 := strings.Index(target, ":/") + // handle unix:addr which will fail with url.Parse + if m1 >= 0 && m2 < 0 { + if n := target[0:m1]; n == "unix" { + return n, target[m1+1:] + } + } + if m2 >= 0 { + t, err := url.Parse(target) + if err != nil { + return net, target + } + scheme := t.Scheme + addr := t.Path + if scheme == "unix" { + if addr == "" { + addr = t.Host + } + return scheme, addr + } + } + return net, target +} diff --git a/internal/net/dialer.go b/internal/net/dialer.go new file mode 100644 index 0000000..8441dd5 --- /dev/null +++ b/internal/net/dialer.go @@ -0,0 +1,36 @@ +package net + +import ( + "net" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +func newDefaultDialer() net.Dialer { + // From `grpc.WithContextDialer` comment: + // + // Note: All supported releases of Go (as of December 2023) override the OS + // defaults for TCP keepalive time and interval to 15s. To enable TCP keepalive + // with OS defaults for keepalive time and interval, use a net.Dialer that sets + // the KeepAlive field to a negative value, and sets the SO_KEEPALIVE socket + // option to true from the Control field. For a concrete example of how to do + // this, see internal.NetDialerWithTCPKeepalive(). + // + // https://github.com/grpc/grpc-go/blob/830135e6c5a351abf75f0c9cfdf978e5df8daeba/dialoptions.go#L432 + // + // From `internal.NetDialerWithTCPKeepalive` comment: + // + // TODO: Once https://github.com/golang/go/issues/62254 lands, and the + // appropriate Go version becomes less than our least supported Go version, we + // should look into using the new API to make things more straightforward. + return net.Dialer{ + KeepAlive: time.Duration(-1), + Control: func(_, _ string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + _ = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_KEEPALIVE, 1) + }) + }, + } +} diff --git a/internal/net/dialer_source.go b/internal/net/dialer_source.go new file mode 100644 index 0000000..e6a142a --- /dev/null +++ b/internal/net/dialer_source.go @@ -0,0 +1,69 @@ +package net + +import ( + "context" + "net" + "sync" + + "git.frostfs.info/TrueCloudLab/multinet" +) + +type DialerSource struct { + guard sync.RWMutex + + c Config + + md multinet.Dialer +} + +func NewDialerSource(c Config) (*DialerSource, error) { + result := &DialerSource{} + if err := result.build(c); err != nil { + return nil, err + } + return result, nil +} + +func (s *DialerSource) build(c Config) error { + if c.Enabled { + mc, err := c.toMultinetConfig() + if err != nil { + return err + } + md, err := multinet.NewDialer(mc) + if err != nil { + return err + } + s.md = md + s.c = c + return nil + } + s.md = nil + s.c = c + return nil +} + +// GrpcContextDialer returns grpc.WithContextDialer func. +// Returns nil if multinet disabled. +func (s *DialerSource) GrpcContextDialer() func(context.Context, string) (net.Conn, error) { + s.guard.RLock() + defer s.guard.RUnlock() + + if s.c.Enabled { + return func(ctx context.Context, address string) (net.Conn, error) { + network, address := parseDialTarget(address) + return s.md.DialContext(ctx, network, address) + } + } + return nil +} + +func (s *DialerSource) Update(c Config) error { + s.guard.Lock() + defer s.guard.Unlock() + + if s.c.equals(c) { + return nil + } + return s.build(c) +} diff --git a/internal/net/event_handler.go b/internal/net/event_handler.go new file mode 100644 index 0000000..9520c01 --- /dev/null +++ b/internal/net/event_handler.go @@ -0,0 +1,28 @@ +package net + +import ( + "net" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "go.uber.org/zap" +) + +type LogEventHandler struct { + logger *zap.Logger +} + +func (l LogEventHandler) DialPerformed(sourceIP net.Addr, _, address string, err error) { + sourceIPString := "undefined" + if sourceIP != nil { + sourceIPString = sourceIP.Network() + "://" + sourceIP.String() + } + if err == nil { + l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), zap.String("destination", address)) + } else { + l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), zap.String("destination", address), zap.Error(err)) + } +} + +func NewLogEventHandler(logger *zap.Logger) LogEventHandler { + return LogEventHandler{logger: logger} +} From 8bc64ce5e997b169b9c146b3ee291b86c3fdab62 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 29 Oct 2024 14:41:49 +0300 Subject: [PATCH 18/86] [#160] Use source dialer for gRPC connection to storage Signed-off-by: Alex Vanin --- cmd/http-gw/app.go | 12 +++++++++-- cmd/http-gw/settings.go | 47 ++++++++++++++++++++++++++++++++++++++++- internal/logs/logs.go | 2 ++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index fa93a0e..84379d4 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -21,6 +21,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" @@ -87,6 +88,7 @@ type ( // appSettings stores reloading parameters, so it has to provide getters and setters which use RWMutex. appSettings struct { reconnectInterval time.Duration + dialerSource *internalnet.DialerSource mu sync.RWMutex defaultTimestamp bool @@ -148,6 +150,8 @@ func newApp(ctx context.Context, opt ...Option) App { opt[i](a) } + a.initAppSettings() + // -- setup FastHTTP server -- a.webServer.Name = "frost-http-gw" a.webServer.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) @@ -161,7 +165,7 @@ func newApp(ctx context.Context, opt ...Option) App { a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - a.pool, a.treePool, a.key = getPools(ctx, a.log, a.cfg) + a.pool, a.treePool, a.key = getPools(ctx, a.log, a.cfg, a.settings.dialerSource) var owner user.ID user.IDFromKey(&owner, a.key.PrivateKey.PublicKey) @@ -169,7 +173,6 @@ func newApp(ctx context.Context, opt ...Option) App { a.setRuntimeParameters() - a.initAppSettings() a.initResolver() a.initMetrics() a.initTracing(ctx) @@ -180,6 +183,7 @@ func newApp(ctx context.Context, opt ...Option) App { func (a *app) initAppSettings() { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.cfg), + dialerSource: getDialerSource(a.log, a.cfg), } a.settings.update(a.cfg, a.log) } @@ -559,6 +563,10 @@ func (a *app) configReload(ctx context.Context) { a.logLevel.SetLevel(lvl) } + if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.cfg, a.log)); err != nil { + a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err)) + } + if err := a.resolver.UpdateResolvers(a.getResolverConfig()); err != nil { a.log.Warn(logs.FailedToUpdateResolvers, zap.Error(err)) } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 5fb628d..f464fbc 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -16,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" @@ -58,6 +59,8 @@ const ( defaultCORSMaxAge = 600 // seconds + defaultMultinetFallbackDelay = 300 * time.Millisecond + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -151,6 +154,13 @@ const ( cfgCORSAllowCredentials = "cors.allow_credentials" cfgCORSMaxAge = "cors.max_age" + // Multinet. + cfgMultinetEnabled = "multinet.enabled" + cfgMultinetBalancer = "multinet.balancer" + cfgMultinetRestrict = "multinet.restrict" + cfgMultinetFallbackDelay = "multinet.fallback_delay" + cfgMultinetSubnets = "multinet.subnets" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -245,6 +255,9 @@ func settings() *viper.Viper { v.SetDefault(cfgResolveNamespaceHeader, defaultNamespaceHeader) v.SetDefault(cfgResolveDefaultNamespaces, []string{"", "root"}) + // multinet + v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay) + // Binding flags if err := v.BindPFlag(cfgPprofEnabled, flags.Lookup(cmdPprof)); err != nil { panic(err) @@ -584,7 +597,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { return servers } -func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { +func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper, dialSource *internalnet.DialerSource) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { key, err := getFrostFSKey(cfg, logger) if err != nil { logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) @@ -643,6 +656,7 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool. interceptors := []grpc.DialOption{ grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), + grpc.WithContextDialer(dialSource.GrpcContextDialer()), } prm.SetGRPCDialOptions(interceptors...) prmTree.SetGRPCDialOptions(interceptors...) @@ -745,3 +759,34 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue return defaultValue } + +func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource { + source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger)) + if err != nil { + logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err)) + } + return source +} + +func fetchMultinetConfig(v *viper.Viper, l *zap.Logger) (cfg internalnet.Config) { + cfg.Enabled = v.GetBool(cfgMultinetEnabled) + cfg.Balancer = v.GetString(cfgMultinetBalancer) + cfg.Restrict = v.GetBool(cfgMultinetRestrict) + cfg.FallbackDelay = v.GetDuration(cfgMultinetFallbackDelay) + cfg.Subnets = make([]internalnet.Subnet, 0, 5) + cfg.EventHandler = internalnet.NewLogEventHandler(l) + + for i := 0; ; i++ { + key := cfgMultinetSubnets + "." + strconv.Itoa(i) + "." + subnet := internalnet.Subnet{} + + subnet.Prefix = v.GetString(key + "mask") + if subnet.Prefix == "" { + break + } + subnet.SourceIPs = v.GetStringSlice(key + "source_ips") + cfg.Subnets = append(cfg.Subnets, subnet) + } + + return +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 7b7ddc1..409f87d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -81,4 +81,6 @@ const ( WarnDuplicateAddress = "duplicate address" MultinetDialSuccess = "multinet dial successful" MultinetDialFail = "multinet dial failed" + FailedToLoadMultinetConfig = "failed to load multinet config" + MultinetConfigWontBeUpdated = "multinet config won't be updated" ) From 821f8c2248040415cb3984e39afd38ff9acebdd5 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 29 Oct 2024 14:44:09 +0300 Subject: [PATCH 19/86] [#160] Add documentation for multinet settings Signed-off-by: Alex Vanin --- config/config.env | 13 +++++++++++++ config/config.yaml | 17 ++++++++++++++++ docs/gate-configuration.md | 40 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/config/config.env b/config/config.env index d2f4a56..4fd8132 100644 --- a/config/config.env +++ b/config/config.env @@ -133,3 +133,16 @@ HTTP_GW_CORS_ALLOW_HEADERS="*" HTTP_GW_CORS_EXPOSE_HEADERS="*" HTTP_GW_CORS_ALLOW_CREDENTIALS=false HTTP_GW_CORS_MAX_AGE=600 + +# Multinet properties +# Enable multinet support +HTTP_GW_MULTINET_ENABLED=false +# Strategy to pick source IP address +HTTP_GW_MULTINET_BALANCER=roundrobin +# Restrict requests with unknown destination subnet +HTTP_GW_MULTINET_RESTRICT=false +# Delay between ipv6 to ipv4 fallback switch +HTTP_GW_MULTINET_FALLBACK_DELAY=300ms +# List of subnets and IP addresses to use as source for those subnets +HTTP_GW_MULTINET_SUBNETS_1_MASK=1.2.3.4/24 +HTTP_GW_MULTINET_SUBNETS_1_SOURCE_IPS=1.2.3.4 1.2.3.5 diff --git a/config/config.yaml b/config/config.yaml index dd985ad..9169acc 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -146,3 +146,20 @@ cors: expose_headers: [] allow_credentials: false max_age: 600 + +# Multinet properties +multinet: + # Enable multinet support + enabled: false + # Strategy to pick source IP address + balancer: roundrobin + # Restrict requests with unknown destination subnet + restrict: false + # Delay between ipv6 to ipv4 fallback switch + fallback_delay: 300ms + # List of subnets and IP addresses to use as source for those subnets + subnets: + - mask: 1.2.3.4/24 + source_ips: + - 1.2.3.4 + - 1.2.3.5 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 7a3eba7..be4b30b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -58,6 +58,7 @@ $ cat http.log | `cache` | [Cache configuration](#cache-section) | | `resolve_bucket` | [Bucket name resolving configuration](#resolve_bucket-section) | | `index_page` | [Index page configuration](#index_page-section) | +| `multinet` | [Multinet configuration](#multinet-section) | # General section @@ -387,3 +388,42 @@ cors: | `expose_headers` | `[]string` | yes | | Values for `Access-Control-Expose-Headers` headers. | | `allow_credentials` | `bool` | yes | `false` | Values for `Access-Control-Allow-Credentials` headers. | | `max_age` | `int` | yes | `600` | Values for `Access-Control-Max-Age ` headers. | + +# `multinet` section + +Configuration of multinet support. + +```yaml +multinet: + enabled: false + balancer: roundrobin + restrict: false + fallback_delay: 300ms + subnets: + - mask: 1.2.3.4/24 + source_ips: + - 1.2.3.4 + - 1.2.3.5 +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|------------------|--------------------------------|---------------|---------------|--------------------------------------------------------------------------------------------| +| `enabled` | `bool` | yes | `false` | Enables multinet setting to manage source ip of outcoming requests. | +| `balancer` | `string` | yes | `""` | Strategy to pick source IP. By default picks first address. Supports `roundrobin` setting. | +| `restrict` | `bool` | yes | `false` | Restricts requests to an undefined subnets. | +| `fallback_delay` | `duration` | yes | `300ms` | Delay between IPv6 and IPv4 fallback stack switch. | +| `subnets` | [[]Subnet](#subnet-subsection) | yes | | Set of subnets to apply multinet dial settings. | + +#### `subnet` subsection + +```yaml +- mask: 1.2.3.4/24 + source_ips: + - 1.2.3.4 + - 1.2.3.5 +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|--------------|------------|---------------|---------------|----------------------------------------------------------------------| +| `mask` | `string` | yes | | Destination subnet. | +| `source_ips` | `[]string` | yes | | Array of source IP addresses to use when dialing destination subnet. | From 679731ee52f72cf060b731d1ccd24e357b121f2c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 5 Nov 2024 17:50:50 +0300 Subject: [PATCH 20/86] [#161] Update SDK Need fix https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pulls/282 Signed-off-by: Denis Kirillov --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index efb79eb..9048009 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.22 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 @@ -24,7 +24,7 @@ require ( go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/trace v1.28.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.26.0 golang.org/x/sys v0.22.0 google.golang.org/grpc v1.66.2 @@ -41,7 +41,7 @@ require ( github.com/Microsoft/hcsshim v0.9.2 // indirect github.com/VictoriaMetrics/easyproto v0.1.4 // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index ce02ee6..b6069d2 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e h1:740ABnOBYx4o6jxULHdSSnVW2fYIO35ohg+Uz59sxd0= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 h1:ivcdxQeQDnx4srF2ezoaeVlF0FAycSAztwfIUJnUI4s= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 h1:ijUci3thz0EwWkuRJDocW5D1RkVAJlt9xNG4CYepC90= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 h1:f7jan6eBDN88DKnKj8GKyWpfjBbSzjDALcDejYKRgCs= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3/go.mod h1:3txOjFJ8M/JFs01h7xOrnQHVn6hZgDNA16ivyUlu1iU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -114,8 +114,8 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -928,8 +928,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From d5b92446bd34ef518f5687b2c607efadbf1ff8d6 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 6 Nov 2024 11:38:03 +0300 Subject: [PATCH 21/86] [#162] Stop using obsolete .github directory This commit is a part of multi-repo cleanup effort: https://git.frostfs.info/TrueCloudLab/frostfs-infra/issues/136 Signed-off-by: Vitaliy Potyarkin --- {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md | 0 {.github => .forgejo}/ISSUE_TEMPLATE/config.yml | 0 {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md | 0 {.github => .forgejo}/logo.svg | 0 .github/CODEOWNERS | 1 - CODEOWNERS | 1 + README.md | 2 +- 7 files changed, 2 insertions(+), 2 deletions(-) rename {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/config.yml (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md (100%) rename {.github => .forgejo}/logo.svg (100%) delete mode 100644 .github/CODEOWNERS create mode 100644 CODEOWNERS diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.forgejo/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.md rename to .forgejo/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.forgejo/ISSUE_TEMPLATE/config.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/config.yml rename to .forgejo/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.forgejo/ISSUE_TEMPLATE/feature_request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .forgejo/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/logo.svg b/.forgejo/logo.svg similarity index 100% rename from .github/logo.svg rename to .forgejo/logo.svg diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index c280648..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @alexvanin @dkirillov diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..43df11e --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +.* @alexvanin @dkirillov diff --git a/README.md b/README.md index 019b8ff..e1af0eb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

-FrostFS logo +FrostFS logo

FrostFS is a decentralized distributed object storage integrated with the NEO Blockchain. From 22d905e51ec963c388aba8930e66d4c4a866aacf Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 15 Nov 2024 14:30:23 +0300 Subject: [PATCH 22/86] [#165] Execute CI on push to master Discussion: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/issues/550 Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/builds.yml | 6 +++++- .forgejo/workflows/tests.yml | 6 +++++- .forgejo/workflows/vulncheck.yml | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows/builds.yml b/.forgejo/workflows/builds.yml index 490a97c..7c2bb04 100644 --- a/.forgejo/workflows/builds.yml +++ b/.forgejo/workflows/builds.yml @@ -1,4 +1,8 @@ -on: [pull_request] +on: + pull_request: + push: + branches: + - master jobs: builds: diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index db7f986..81d93dc 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -1,4 +1,8 @@ -on: [pull_request] +on: + pull_request: + push: + branches: + - master jobs: lint: diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 7a82bc3..76e2965 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -1,4 +1,8 @@ -on: [pull_request] +on: + pull_request: + push: + branches: + - master jobs: vulncheck: From 9c0b499ea6b8ea7ef8bdbadd254415b46b579b9a Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Fri, 15 Nov 2024 10:53:46 +0300 Subject: [PATCH 23/86] [#164] Add tracing attributes Signed-off-by: Roman Loginov --- cmd/http-gw/app.go | 7 +++++++ cmd/http-gw/settings.go | 33 +++++++++++++++++++++++++++++---- config/config.env | 4 ++++ config/config.yaml | 6 ++++++ docs/gate-configuration.md | 34 ++++++++++++++++++++++++++++------ go.mod | 2 +- go.sum | 4 ++-- 7 files changed, 77 insertions(+), 13 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 84379d4..853977c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -927,6 +927,13 @@ func (a *app) initTracing(ctx context.Context) { cfg.ServerCaCertPool = certPool } + attributes, err := fetchTracingAttributes(a.cfg) + if err != nil { + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + return + } + cfg.Attributes = attributes + updated, err := tracing.Setup(ctx, cfg) if err != nil { a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index f464fbc..4f1712b 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -86,10 +86,11 @@ const ( cfgPprofAddress = "pprof.address" // Tracing ... - cfgTracingEnabled = "tracing.enabled" - cfgTracingExporter = "tracing.exporter" - cfgTracingEndpoint = "tracing.endpoint" - cfgTracingTrustedCa = "tracing.trusted_ca" + cfgTracingEnabled = "tracing.enabled" + cfgTracingExporter = "tracing.exporter" + cfgTracingEndpoint = "tracing.endpoint" + cfgTracingTrustedCa = "tracing.trusted_ca" + cfgTracingAttributes = "tracing.attributes" // Pool config. cfgConTimeout = "connect_timeout" @@ -790,3 +791,27 @@ func fetchMultinetConfig(v *viper.Viper, l *zap.Logger) (cfg internalnet.Config) return } + +func fetchTracingAttributes(v *viper.Viper) (map[string]string, error) { + attributes := make(map[string]string) + for i := 0; ; i++ { + key := cfgTracingAttributes + "." + strconv.Itoa(i) + "." + attrKey := v.GetString(key + "key") + attrValue := v.GetString(key + "value") + if attrKey == "" { + break + } + + if _, ok := attributes[attrKey]; ok { + return nil, fmt.Errorf("tracing attribute key %s defined more than once", attrKey) + } + + if attrValue == "" { + return nil, fmt.Errorf("empty tracing attribute value for key %s", attrKey) + } + + attributes[attrKey] = attrValue + } + + return attributes, nil +} diff --git a/config/config.env b/config/config.env index 4fd8132..db54c92 100644 --- a/config/config.env +++ b/config/config.env @@ -104,6 +104,10 @@ HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_EXPORTER="otlp_grpc" HTTP_GW_TRACING_TRUSTED_CA="" +HTTP_GW_TRACING_ATTRIBUTES_0_KEY=key0 +HTTP_GW_TRACING_ATTRIBUTES_0_VALUE=value +HTTP_GW_TRACING_ATTRIBUTES_1_KEY=key1 +HTTP_GW_TRACING_ATTRIBUTES_1_VALUE=value HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/config.yaml b/config/config.yaml index 9169acc..6c89e78 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -9,11 +9,17 @@ pprof: prometheus: enabled: false # Enable metrics. address: localhost:8084 + tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost:4317" trusted_ca: "" + attributes: + - key: key0 + value: value + - key: key1 + value: value logger: level: debug # Log level. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index be4b30b..5b5b018 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -268,14 +268,36 @@ tracing: exporter: "otlp_grpc" endpoint: "localhost:4317" trusted_ca: "/etc/ssl/telemetry-trusted-ca.pem" + attributes: + - key: key0 + value: value + - key: key1 + value: value ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|--------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| -| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | -| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | -| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | -| `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | +| Parameter | Type | SIGHUP reload | Default value | Description | +| ------------ | -------------------------------------- | ------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | +| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | +| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | +| `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | +| `attributes` | [[]Attributes](#attributes-subsection) | yes | | An array of configurable attributes in key-value format. | + + +#### `attributes` subsection + +```yaml + attributes: + - key: key0 + value: value + - key: key1 + value: value +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------------|----------|---------------|---------------|----------------------------------------------------------| +| `key` | `string` | yes | | Attribute key. | +| `value` | `string` | yes | | Attribute value. | # `runtime` section Contains runtime parameters. diff --git a/go.mod b/go.mod index 9048009..ca8f4fe 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 diff --git a/go.sum b/go.sum index b6069d2..197f21c 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 h1:f7jan6eBDN88DKnKj8GKyWpfjBbSzjDALcDejYKRgCs= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3/go.mod h1:3txOjFJ8M/JFs01h7xOrnQHVn6hZgDNA16ivyUlu1iU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= From 43764772aa6f90fbeaf99cd6bccdebc157b5330a Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Thu, 10 Oct 2024 11:59:53 +0300 Subject: [PATCH 24/86] [#151] index page: Add browse via native protocol Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 20 ++- cmd/http-gw/settings.go | 6 +- config/config.env | 9 + config/config.yaml | 3 + docs/gate-configuration.md | 30 ++-- go.mod | 1 + go.sum | 2 + internal/handler/browse.go | 300 ++++++++++++++++++++++++++----- internal/handler/download.go | 23 +-- internal/handler/handler.go | 87 +++++---- internal/handler/handler_test.go | 12 +- internal/handler/head.go | 4 +- internal/handler/utils.go | 21 +-- internal/logs/logs.go | 6 +- internal/templates/index.gotmpl | 42 +++-- tree/tree.go | 104 ++++++++++- 16 files changed, 537 insertions(+), 133 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 853977c..0dd53a6 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -40,6 +40,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" + "github.com/panjf2000/ants/v2" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.opentelemetry.io/otel/trace" @@ -89,6 +90,7 @@ type ( appSettings struct { reconnectInterval time.Duration dialerSource *internalnet.DialerSource + workerPoolSize int mu sync.RWMutex defaultTimestamp bool @@ -184,6 +186,7 @@ func (a *app) initAppSettings() { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.cfg), dialerSource: getDialerSource(a.log, a.cfg), + workerPoolSize: a.cfg.GetInt(cfgWorkerPoolSize), } a.settings.update(a.cfg, a.log) } @@ -490,7 +493,13 @@ func (a *app) setHealthStatus() { } func (a *app) Serve() { - handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool))) + workerPool := a.initWorkerPool() + defer func() { + workerPool.Release() + close(a.webDone) + }() + + handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) // Configure router. a.configureRouter(handler) @@ -532,8 +541,14 @@ LOOP: a.metrics.Shutdown() a.stopServices() a.shutdownTracing() +} - close(a.webDone) +func (a *app) initWorkerPool() *ants.Pool { + workerPool, err := ants.NewPool(a.settings.workerPoolSize) + if err != nil { + a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err)) + } + return workerPool } func (a *app) shutdownTracing() { @@ -609,6 +624,7 @@ func (a *app) stopServices() { svc.ShutDown(ctx) } } + func (a *app) configureRouter(handler *handler.Handler) { r := router.New() r.RedirectTrailingSlash = true diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 4f1712b..316c500 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -71,6 +71,8 @@ const ( cfgIndexPageEnabled = "index_page.enabled" cfgIndexPageTemplatePath = "index_page.template_path" + cfgWorkerPoolSize = "worker_pool_size" + // Web. cfgWebReadBufferSize = "web.read_buffer_size" cfgWebWriteBufferSize = "web.write_buffer_size" @@ -228,9 +230,6 @@ func settings() *viper.Viper { // pool: v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) - v.SetDefault(cfgIndexPageEnabled, false) - v.SetDefault(cfgIndexPageTemplatePath, "") - // frostfs: v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) @@ -242,6 +241,7 @@ func settings() *viper.Viper { v.SetDefault(cfgWebStreamRequestBody, true) v.SetDefault(cfgWebMaxRequestBodySize, fasthttp.DefaultMaxRequestBodySize) + v.SetDefault(cfgWorkerPoolSize, 1000) // upload header v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) diff --git a/config/config.env b/config/config.env index db54c92..fd51392 100644 --- a/config/config.env +++ b/config/config.env @@ -150,3 +150,12 @@ HTTP_GW_MULTINET_FALLBACK_DELAY=300ms # List of subnets and IP addresses to use as source for those subnets HTTP_GW_MULTINET_SUBNETS_1_MASK=1.2.3.4/24 HTTP_GW_MULTINET_SUBNETS_1_SOURCE_IPS=1.2.3.4 1.2.3.5 + +# Number of workers in handler's worker pool +HTTP_GW_WORKER_POOL_SIZE=1000 + +# Index page +# Enable index page support +HTTP_GW_INDEX_PAGE_ENABLED=false +# Index page template path +HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl \ No newline at end of file diff --git a/config/config.yaml b/config/config.yaml index 6c89e78..ef5c529 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -113,6 +113,9 @@ request_timeout: 5s # Timeout to check node health during rebalance. rebalance_timer: 30s # Interval to check nodes health. pool_error_threshold: 100 # The number of errors on connection after which node is considered as unhealthy. +# Number of workers in handler's worker pool +worker_pool_size: 1000 + # Enable index page to see objects list for specified container and prefix index_page: enabled: false diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 5b5b018..c6cb617 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -75,18 +75,21 @@ request_timeout: 5s rebalance_timer: 30s pool_error_threshold: 100 reconnect_interval: 1m +worker_pool_size: 1000 + ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|------------------------|------------|---------------|---------------|-------------------------------------------------------------------------------------------------| -| `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | -| `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | -| `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | -| `stream_timeout` | `duration` | | `10s` | Timeout for individual operations in streaming RPC. | -| `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | -| `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | -| `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | -| `reconnect_interval` | `duration` | no | `1m` | Listeners reconnection interval. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|------------------------|------------|---------------|---------------|------------------------------------------------------------------------------------| +| `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | +| `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | +| `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | +| `stream_timeout` | `duration` | | `10s` | Timeout for individual operations in streaming RPC. | +| `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | +| `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | +| `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | +| `reconnect_interval` | `duration` | no | `1m` | Listeners reconnection interval. | +| `worker_pool_size` | `int` | no | `1000` | Maximum worker count in handler's worker pool. | # `wallet` section @@ -374,7 +377,12 @@ resolve_bucket: # `index_page` section -Parameters for index HTML-page output with S3-bucket or S3-subdir content for `Get object` request +Parameters for index HTML-page output. Activates if `GetObject` request returns `not found`. Two +index page modes available: + +* `s3` mode uses tree service for listing objects, +* `native` sends requests to nodes via native protocol. + If request pass S3-bucket name instead of CID, `s3` mode will be used, otherwise `native`. ```yaml index_page: diff --git a/go.mod b/go.mod index ca8f4fe..a2f41d8 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/docker/go-units v0.4.0 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.106.2 + github.com/panjf2000/ants/v2 v2.5.0 github.com/prometheus/client_golang v1.19.0 github.com/prometheus/client_model v0.5.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 197f21c..a7a5be4 100644 --- a/go.sum +++ b/go.sum @@ -682,6 +682,8 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q= +github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= diff --git a/internal/handler/browse.go b/internal/handler/browse.go index c89d8c5..b24a569 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -1,15 +1,21 @@ package handler import ( + "context" "html/template" "net/url" "sort" "strconv" "strings" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/docker/go-units" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -25,19 +31,68 @@ const ( type ( BrowsePageData struct { - BucketName, - Prefix string - Objects []ResponseObject + HasErrors bool + Container string + Prefix string + Protocol string + Objects []ResponseObject } ResponseObject struct { OID string Created string FileName string + FilePath string Size string IsDir bool + GetURL string } ) +func newListObjectsResponseS3(attrs map[string]string) ResponseObject { + return ResponseObject{ + Created: formatTimestamp(attrs[attrCreated]), + OID: attrs[attrOID], + FileName: attrs[attrFileName], + Size: attrs[attrSize], + IsDir: attrs[attrOID] == "", + } +} + +func newListObjectsResponseNative(attrs map[string]string) ResponseObject { + filename := lastPathElement(attrs[object.AttributeFilePath]) + if filename == "" { + filename = attrs[attrFileName] + } + return ResponseObject{ + OID: attrs[attrOID], + Created: formatTimestamp(attrs[object.AttributeTimestamp] + "000"), + FileName: filename, + FilePath: attrs[object.AttributeFilePath], + Size: attrs[attrSize], + IsDir: false, + } +} + +func getNextDir(filepath, prefix string) string { + restPath := strings.Replace(filepath, prefix, "", 1) + index := strings.Index(restPath, "/") + if index == -1 { + return "" + } + return restPath[:index] +} + +func lastPathElement(path string) string { + if path == "" { + return path + } + index := strings.LastIndex(path, "/") + if index == len(path)-1 { + index = strings.LastIndex(path[:index], "/") + } + return path[index+1:] +} + func parseTimestamp(tstamp string) (time.Time, error) { millis, err := strconv.ParseInt(tstamp, 10, 64) if err != nil { @@ -47,16 +102,6 @@ func parseTimestamp(tstamp string) (time.Time, error) { return time.UnixMilli(millis), nil } -func NewResponseObject(nodes map[string]string) ResponseObject { - return ResponseObject{ - OID: nodes[attrOID], - Created: nodes[attrCreated], - FileName: nodes[attrFileName], - Size: nodes[attrSize], - IsDir: nodes[attrOID] == "", - } -} - func formatTimestamp(strdate string) string { date, err := parseTimestamp(strdate) if err != nil || date.IsZero() { @@ -94,12 +139,9 @@ func trimPrefix(encPrefix string) string { return prefix[:slashIndex] } -func urlencode(prefix, filename string) string { +func urlencode(path string) string { var res strings.Builder - path := filename - if prefix != "" { - path = strings.Join([]string{prefix, filename}, "/") - } + prefixParts := strings.Split(path, "/") for _, prefixPart := range prefixParts { prefixPart = "/" + url.PathEscape(prefixPart) @@ -112,46 +154,220 @@ func urlencode(prefix, filename string) string { return res.String() } -func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) { +type GetObjectsResponse struct { + objects []ResponseObject + hasErrors bool +} + +func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { + nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) + if err != nil { + return nil, err + } + + result := &GetObjectsResponse{ + objects: make([]ResponseObject, 0, len(nodes)), + } + for _, node := range nodes { + meta := node.GetMeta() + if meta == nil { + continue + } + var attrs = make(map[string]string, len(meta)) + for _, m := range meta { + attrs[m.GetKey()] = string(m.GetValue()) + } + obj := newListObjectsResponseS3(attrs) + obj.FilePath = prefix + obj.FileName + obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath) + result.objects = append(result.objects, obj) + } + + return result, nil +} + +func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { + var basePath string + if ind := strings.LastIndex(prefix, "/"); ind != -1 { + basePath = prefix[:ind+1] + } + + filters := object.NewSearchFilters() + filters.AddRootFilter() + if prefix != "" { + filters.AddFilter(object.AttributeFilePath, prefix, object.MatchCommonPrefix) + } + + prm := PrmObjectSearch{ + PrmAuth: PrmAuth{ + BearerToken: bearerToken(ctx), + }, + Container: bucketInfo.CID, + Filters: filters, + } + objectIDs, err := h.frostfs.SearchObjects(ctx, prm) + if err != nil { + return nil, err + } + defer objectIDs.Close() + + resp, err := h.headDirObjects(ctx, bucketInfo.CID, objectIDs, basePath) + if err != nil { + return nil, err + } + + log := utils.GetReqLogOrDefault(ctx, h.log) + dirs := make(map[string]struct{}) + result := &GetObjectsResponse{ + objects: make([]ResponseObject, 0, 100), + } + for objExt := range resp { + if objExt.Error != nil { + log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error)) + result.hasErrors = true + continue + } + if objExt.Object.IsDir { + if _, ok := dirs[objExt.Object.FileName]; ok { + continue + } + objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + urlencode(objExt.Object.FilePath) + dirs[objExt.Object.FileName] = struct{}{} + } else { + objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + objExt.Object.OID + } + result.objects = append(result.objects, objExt.Object) + } + return result, nil +} + +type ResponseObjectExtended struct { + Object ResponseObject + Error error +} + +func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs ResObjectSearch, basePath string) (<-chan ResponseObjectExtended, error) { + res := make(chan ResponseObjectExtended) + + go func() { + defer close(res) + log := utils.GetReqLogOrDefault(ctx, h.log).With( + zap.String("cid", cnrID.EncodeToString()), + zap.String("path", basePath), + ) + var wg sync.WaitGroup + err := objectIDs.Iterate(func(id oid.ID) bool { + wg.Add(1) + err := h.workerPool.Submit(func() { + defer wg.Done() + var obj ResponseObjectExtended + obj.Object, obj.Error = h.headDirObject(ctx, cnrID, id, basePath) + res <- obj + }) + if err != nil { + wg.Done() + log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err)) + } + select { + case <-ctx.Done(): + return true + default: + return false + } + }) + if err != nil { + log.Error(logs.FailedToIterateOverResponse, zap.Error(err)) + } + wg.Wait() + }() + + return res, nil +} + +func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID, basePath string) (ResponseObject, error) { + addr := newAddress(cnrID, objID) + obj, err := h.frostfs.HeadObject(ctx, PrmObjectHead{ + PrmAuth: PrmAuth{BearerToken: bearerToken(ctx)}, + Address: addr, + }) + if err != nil { + return ResponseObject{}, err + } + + attrs := loadAttributes(obj.Attributes()) + attrs[attrOID] = objID.EncodeToString() + if multipartSize, ok := attrs[attributeMultipartObjectSize]; ok { + attrs[attrSize] = multipartSize + } else { + attrs[attrSize] = strconv.FormatUint(obj.PayloadSize(), 10) + } + + dirname := getNextDir(attrs[object.AttributeFilePath], basePath) + if dirname == "" { + return newListObjectsResponseNative(attrs), nil + } + + return ResponseObject{ + FileName: dirname, + FilePath: basePath + dirname, + IsDir: true, + }, nil +} + +type browseParams struct { + bucketInfo *data.BucketInfo + prefix string + isNative bool + listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error) +} + +func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) { + const S3Protocol = "s3" + const FrostfsProtocol = "frostfs" + ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With(zap.String("bucket", bucketInfo.Name)) - - nodes, err := h.listObjects(ctx, bucketInfo, prefix) + log := reqLog.With( + zap.String("bucket", p.bucketInfo.Name), + zap.String("container", p.bucketInfo.CID.EncodeToString()), + zap.String("prefix", p.prefix), + ) + resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix) if err != nil { logAndSendBucketError(c, log, err) return } - respObjects := make([]ResponseObject, len(nodes)) - - for i, node := range nodes { - respObjects[i] = NewResponseObject(node) - } - - sort.Slice(respObjects, func(i, j int) bool { - if respObjects[i].IsDir == respObjects[j].IsDir { - return respObjects[i].FileName < respObjects[j].FileName + objects := resp.objects + sort.Slice(objects, func(i, j int) bool { + if objects[i].IsDir == objects[j].IsDir { + return objects[i].FileName < objects[j].FileName } - return respObjects[i].IsDir + return objects[i].IsDir }) - indexTemplate := h.config.IndexPageTemplate() tmpl, err := template.New("index").Funcs(template.FuncMap{ - "formatTimestamp": formatTimestamp, - "formatSize": formatSize, - "trimPrefix": trimPrefix, - "urlencode": urlencode, - "parentDir": parentDir, - }).Parse(indexTemplate) + "formatSize": formatSize, + "trimPrefix": trimPrefix, + "urlencode": urlencode, + "parentDir": parentDir, + }).Parse(h.config.IndexPageTemplate()) if err != nil { logAndSendBucketError(c, log, err) return } + bucketName := p.bucketInfo.Name + protocol := S3Protocol + if p.isNative { + bucketName = p.bucketInfo.CID.EncodeToString() + protocol = FrostfsProtocol + } if err = tmpl.Execute(c, &BrowsePageData{ - BucketName: bucketInfo.Name, - Prefix: prefix, - Objects: respObjects, + Container: bucketName, + Prefix: p.prefix, + Objects: objects, + Protocol: protocol, + HasErrors: resp.hasErrors, }); err != nil { logAndSendBucketError(c, log, err) return diff --git a/internal/handler/download.go b/internal/handler/download.go index 19380d4..cd4e55a 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -23,13 +23,16 @@ import ( // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { - test, _ := c.UserValue("oid").(string) - var id oid.ID - err := id.DecodeString(test) - if err != nil { - h.byObjectName(c, h.receiveFile) - } else { - h.byAddress(c, h.receiveFile) + oidURLParam := c.UserValue("oid").(string) + downloadQueryParam := c.QueryArgs().GetBool("download") + + switch { + case isObjectID(oidURLParam): + h.byNativeAddress(c, h.receiveFile) + case !isContainerRoot(oidURLParam) && (downloadQueryParam || !isDir(oidURLParam)): + h.byS3Path(c, h.receiveFile) + default: + h.browseIndex(c) } } @@ -45,7 +48,7 @@ func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { h.byAttribute(c, h.receiveFile) } -func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) { +func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) { filters := object.NewSearchFilters() filters.AddRootFilter() filters.AddFilter(key, val, op) @@ -54,7 +57,7 @@ func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op PrmAuth: PrmAuth{ BearerToken: bearerToken(ctx), }, - Container: *cnrID, + Container: cnrID, Filters: filters, } @@ -102,7 +105,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { return } - resSearch, err := h.search(ctx, &bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) + resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 62d0897..9ed7f99 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -22,6 +22,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -165,6 +166,7 @@ type Handler struct { containerResolver ContainerResolver tree *tree.Tree cache *cache.BucketCache + workerPool *ants.Pool } type AppParams struct { @@ -175,7 +177,7 @@ type AppParams struct { Cache *cache.BucketCache } -func New(params *AppParams, config Config, tree *tree.Tree) *Handler { +func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, @@ -184,14 +186,15 @@ func New(params *AppParams, config Config, tree *tree.Tree) *Handler { containerResolver: params.Resolver, tree: tree, cache: params.Cache, + workerPool: workerPool, } } -// byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that +// byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { idCnr, _ := c.UserValue("cid").(string) - idObj, _ := c.UserValue("oid").(string) + idObj, _ := url.PathUnescape(c.UserValue("oid").(string)) ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) @@ -215,12 +218,11 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ f(ctx, *h.newRequest(c, log), addr) } -// byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that -// prepares request and object address to it. -func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +// byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that +// resolves object address from S3-like path /. +func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { bucketname := c.UserValue("cid").(string) key := c.UserValue("oid").(string) - download := c.QueryArgs().GetBool("download") ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) @@ -239,15 +241,6 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r } foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey) - if h.config.IndexPageEnabled() && !download && string(c.Method()) != fasthttp.MethodHead { - if isDir(unescapedKey) || isContainerRoot(unescapedKey) { - if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK { - c.SetStatusCode(code) - h.browseObjects(c, bktInfo, unescapedKey) - return - } - } - } if err != nil { if errors.Is(err, tree.ErrNodeAccessDenied) { response.Error(c, "Access Denied", fasthttp.StatusForbidden) @@ -267,7 +260,7 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r f(ctx, *h.newRequest(c, log), addr) } -// byAttribute is a wrapper similar to byAddress. +// byAttribute is a wrapper similar to byNativeAddress. func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { scid, _ := c.UserValue("cid").(string) key, _ := c.UserValue("attr_key").(string) @@ -298,7 +291,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re return } - res, err := h.search(ctx, &bktInfo.CID, key, val, object.MatchStringEqual) + res, err := h.search(ctx, bktInfo.CID, key, val, object.MatchStringEqual) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -395,24 +388,50 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } -func (h *Handler) listObjects(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]map[string]string, error) { - nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) +func (h *Handler) browseIndex(c *fasthttp.RequestCtx) { + if !h.config.IndexPageEnabled() { + c.SetStatusCode(fasthttp.StatusNotFound) + return + } + + cidURLParam := c.UserValue("cid").(string) + oidURLParam := c.UserValue("oid").(string) + + ctx := utils.GetContextFromRequest(c) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("cid", cidURLParam), zap.String("oid", oidURLParam)) + + unescapedKey, err := url.QueryUnescape(oidURLParam) if err != nil { - return nil, err + logAndSendBucketError(c, log, err) + return } - var objects = make([]map[string]string, 0, len(nodes)) - for _, node := range nodes { - meta := node.GetMeta() - if meta == nil { - continue - } - var obj = make(map[string]string, len(meta)) - for _, m := range meta { - obj[m.GetKey()] = string(m.GetValue()) - } - objects = append(objects, obj) + bktInfo, err := h.getBucketInfo(ctx, cidURLParam, log) + if err != nil { + logAndSendBucketError(c, log, err) + return } - return objects, nil + listFunc := h.getDirObjectsS3 + isNativeList := false + + err = h.tree.CheckSettingsNodeExist(ctx, bktInfo) + if err != nil { + if errors.Is(err, tree.ErrNodeNotFound) { + // tree probe failed, try to use native + listFunc = h.getDirObjectsNative + isNativeList = true + } else { + logAndSendBucketError(c, log, err) + return + } + } + + h.browseObjects(c, browseParams{ + bucketInfo: bktInfo, + prefix: unescapedKey, + listObjects: listFunc, + isNative: isNativeList, + }) } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 4fe9153..34668a5 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -26,6 +26,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -57,10 +58,11 @@ func (c *configMock) IndexPageEnabled() bool { return false } -func (c *configMock) IndexPageTemplatePath() string { +func (c *configMock) IndexPageTemplate() string { return "" } -func (c *configMock) IndexPageTemplate() string { + +func (c *configMock) IndexPageNativeTemplate() string { return "" } @@ -126,7 +128,11 @@ func prepareHandlerContext() (*handlerContext, error) { treeMock := &treeClientMock{} cfgMock := &configMock{} - handler := New(params, cfgMock, tree.NewTree(treeMock)) + workerPool, err := ants.NewPool(1000) + if err != nil { + return nil, err + } + handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool) return &handlerContext{ key: key, diff --git a/internal/handler/head.go b/internal/handler/head.go index f0a1e94..ccd6a91 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -107,9 +107,9 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { err := id.DecodeString(test) if err != nil { - h.byObjectName(c, h.headObject) + h.byS3Path(c, h.headObject) } else { - h.byAddress(c, h.headObject) + h.byNativeAddress(c, h.headObject) } } diff --git a/internal/handler/utils.go b/internal/handler/utils.go index a944b67..b537d64 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -2,17 +2,16 @@ package handler import ( "context" - "errors" "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -46,19 +45,21 @@ func isDir(name string) bool { return strings.HasSuffix(name, "/") } +func isObjectID(s string) bool { + var objID oid.ID + return objID.DecodeString(s) == nil +} + func isContainerRoot(key string) bool { return key == "" } -func checkErrorType(err error) int { - switch { - case err == nil: - return fasthttp.StatusOK - case errors.Is(err, tree.ErrNodeAccessDenied): - return fasthttp.StatusForbidden - default: - return fasthttp.StatusNotFound +func loadAttributes(attrs []object.Attribute) map[string]string { + result := make(map[string]string) + for _, attr := range attrs { + result[attr.Key()] = attr.Value() } + return result } func isValidToken(s string) bool { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 409f87d..4dfa21f 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -31,7 +31,8 @@ const ( CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go - FailedToReadIndexPageTemplate = "failed to read index page template, set default" // Warn in ../../app.go + FailedToCreateWorkerPool = "failed to create worker pool" // Fatal in ../../app.go + FailedToReadIndexPageTemplate = "failed to read index page template" // Error in ../../app.go SetCustomIndexPageTemplate = "set custom index page template" // Info in ../../app.go ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go @@ -71,6 +72,9 @@ const ( AddedStoragePeer = "added storage peer" // Info in ../../settings.go CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go + FailedToSumbitTaskToPool = "failed to submit task to pool" // Error in ../handler/browse.go + FailedToHeadObject = "failed to head object" // Error in ../handler/browse.go + FailedToIterateOverResponse = "failed to iterate over search response" // Error in ../handler/browse.go InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go diff --git a/internal/templates/index.gotmpl b/internal/templates/index.gotmpl index ea66a62..b14cc06 100644 --- a/internal/templates/index.gotmpl +++ b/internal/templates/index.gotmpl @@ -1,11 +1,20 @@ -{{$bucketName := .BucketName}} +{{$container := .Container}} {{ $prefix := trimPrefix .Prefix }} - Index of s3://{{$bucketName}}/{{if $prefix}}/{{$prefix}}/{{end}} + Index of {{.Protocol}}://{{$container}} + /{{if $prefix}}/{{$prefix}}/{{end}} -

Index of s3://{{$bucketName}}/{{if $prefix}}{{$prefix}}/{{end}}

+

Index of {{.Protocol}}://{{$container}}/{{if $prefix}}{{$prefix}}/{{end}}

+{{ if .HasErrors }} +
+ Errors occurred while processing the request. Perhaps some objects are missing +
+{{ end }} + @@ -42,20 +61,22 @@ {{if $trimmedPrefix }} + {{else}} + {{end}} {{range .Objects}} @@ -63,21 +84,22 @@ + - + - {{ $trimmedPrefix := trimPrefix $prefix }} - {{if $trimmedPrefix }} + {{ $parentPrefix := getParent .Prefix }} + {{if $parentPrefix }} diff --git a/tree/tree.go b/tree/tree.go index 2ee9356..d99e24b 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -7,7 +7,6 @@ import ( "strings" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -52,10 +51,10 @@ type ( var ( // ErrNodeNotFound is returned from ServiceClient in case of not found error. - ErrNodeNotFound = layer.ErrNodeNotFound + ErrNodeNotFound = errors.New("not found") // ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error. - ErrNodeAccessDenied = layer.ErrNodeAccessDenied + ErrNodeAccessDenied = errors.New("access denied") ) const ( @@ -259,7 +258,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name nodes = filterMultipartNodes(nodes) if len(nodes) == 0 { - return nil, layer.ErrNodeNotFound + return nil, ErrNodeNotFound } if len(nodes) != 1 { c.reqLogger(ctx).Warn(logs.FoundSeveralSystemTreeNodes, zap.String("name", name), logs.TagField(logs.TagExternalStorageTree)) @@ -303,7 +302,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { } if targetIndexNode == -1 { - return nil, fmt.Errorf("latest version: %w", layer.ErrNodeNotFound) + return nil, fmt.Errorf("latest version: %w", ErrNodeNotFound) } return nodes[targetIndexNode], nil @@ -324,20 +323,23 @@ func pathFromName(objectName string) []string { return strings.Split(objectName, separator) } -func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) { +func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) { ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix") defer span.End() - rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) + rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator)) if err != nil { - return nil, "", err + if errors.Is(err, ErrNodeNotFound) { + return nil, nil + } + return nil, err } subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) if err != nil { if errors.Is(err, ErrNodeNotFound) { - return nil, "", nil + return nil, nil } - return nil, "", err + return nil, err } nodesMap := make(map[string][]NodeResponse, len(subTree)) @@ -347,10 +349,6 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, } fileName := GetFilename(node) - if !strings.HasPrefix(fileName, tailPrefix) { - continue - } - nodes := nodesMap[fileName] // Add all nodes if flag latestOnly is false. @@ -374,7 +372,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, result = append(result, nodeResponseToNodeInfo(nodes)...) } - return result, strings.TrimSuffix(prefix, tailPrefix), nil + return result, nil } func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo { @@ -386,22 +384,6 @@ func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo { return nodesInfo } -func (c *Tree) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) ([]uint64, string, error) { - rootID := []uint64{0} - path := strings.Split(prefix, separator) - tailPrefix := path[len(path)-1] - - if len(path) > 1 { - var err error - rootID, err = c.getPrefixNodeID(ctx, bktInfo, treeID, path[:len(path)-1]) - if err != nil { - return nil, "", err - } - } - - return rootID, tailPrefix, nil -} - func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefixPath []string) ([]uint64, error) { p := &GetNodesParams{ CnrID: bktInfo.CID, @@ -424,7 +406,7 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr } if len(intermediateNodes) == 0 { - return nil, layer.ErrNodeNotFound + return nil, ErrNodeNotFound } return intermediateNodes, nil From 96a22d98f206ce4910d69ce68da221802cb23c22 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 25 Apr 2025 10:03:16 +0300 Subject: [PATCH 86/86] [#232] Use contract to get container info Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 28 ++++++- cmd/http-gw/settings.go | 8 ++ config/config.env | 3 + config/config.yaml | 5 ++ docs/gate-configuration.md | 13 ++++ go.mod | 2 +- internal/handler/container.go | 42 +++++++++++ internal/handler/frostfs_mock.go | 10 +++ internal/handler/handler.go | 47 +++--------- internal/handler/handler_test.go | 2 +- internal/logs/logs.go | 6 +- .../service/contracts/container/client.go | 73 +++++++++++++++++++ internal/service/contracts/util/util.go | 34 +++++++++ 13 files changed, 229 insertions(+), 44 deletions(-) create mode 100644 internal/handler/container.go create mode 100644 internal/service/contracts/container/client.go create mode 100644 internal/service/contracts/util/util.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index f603d3b..4a83caf 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -22,6 +22,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" + containerClient "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/contracts/container" + contractsUtil "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/contracts/util" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" @@ -39,6 +41,7 @@ import ( "github.com/nspcc-dev/neo-go/cli/flags" "github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/panjf2000/ants/v2" @@ -276,6 +279,14 @@ func (a *app) initContainers(ctx context.Context) { a.corsCnrID = *corsCnrID } +func (a *app) initRPCClient(ctx context.Context) *rpcclient.Client { + rpcCli, err := rpcclient.New(ctx, a.config().GetString(cfgRPCEndpoint), rpcclient.Options{}) + if err != nil { + a.log.Fatal(logs.InitRPCClientFailed, zap.Error(err), logs.TagField(logs.TagApp)) + } + return rpcCli +} + func (a *app) initAppSettings(lc *logLevelConfig) { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.config()), @@ -750,7 +761,22 @@ func (a *app) stopServices() { } func (a *app) configureRouter(workerPool *ants.Pool) { - a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool) + rpcCli := a.initRPCClient(a.ctx) + cnrContractName := a.config().GetString(cfgContractsContainerName) + rpcEndpoint := a.config().GetString(cfgRPCEndpoint) + cnrAddr, err := contractsUtil.ResolveContractHash(cnrContractName, rpcEndpoint) + if err != nil { + a.log.Fatal(logs.FailedToResolveContractHash, zap.Error(err), logs.TagField(logs.TagApp)) + } + cnrClient, err := containerClient.New(containerClient.Config{ + ContractHash: cnrAddr, + Key: a.key, + RPCClient: rpcCli, + }) + if err != nil { + a.log.Fatal(logs.InitContainerContractFailed, zap.Error(err), logs.TagField(logs.TagApp)) + } + a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), cnrClient, workerPool) r := router.New() r.RedirectTrailingSlash = true diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 07722de..4071969 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -62,6 +62,8 @@ const ( defaultMultinetFallbackDelay = 300 * time.Millisecond + defaultContainerContractName = "container.frostfs" + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -197,6 +199,9 @@ const ( cmdConfig = "config" cmdConfigDir = "config-dir" cmdListenAddress = "listen_address" + + // Contracts. + cfgContractsContainerName = "contracts.container.name" ) var ignore = map[string]struct{}{ @@ -401,6 +406,9 @@ func setDefaults(v *viper.Viper, flags *pflag.FlagSet) { // multinet v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay) + // contracts + v.SetDefault(cfgContractsContainerName, defaultContainerContractName) + if resolveMethods, err := flags.GetStringSlice(cfgResolveOrder); err == nil { v.SetDefault(cfgResolveOrder, resolveMethods) } diff --git a/config/config.env b/config/config.env index a86f3e8..ff880d5 100644 --- a/config/config.env +++ b/config/config.env @@ -181,3 +181,6 @@ HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true # Containers properties HTTP_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj + +# Container contract hash (LE) or name in NNS. +HTTP_GW_CONTRACTS_CONTAINER_NAME=container.frostfs diff --git a/config/config.yaml b/config/config.yaml index bb01d47..9b4b3c9 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -199,3 +199,8 @@ features: containers: cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj + +contracts: + container: + # Container contract hash (LE) or name in NNS. + name: container.frostfs diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 08e2679..7f3c4ef 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -60,6 +60,7 @@ $ cat http.log | `multinet` | [Multinet configuration](#multinet-section) | | `features` | [Features configuration](#features-section) | | `containers` | [Containers configuration](#containers-section) | +| `contracts` | [Contracts configuration](#contracts-section) | # General section @@ -527,3 +528,15 @@ containers: | Parameter | Type | SIGHUP reload | Default value | Description | |-----------|----------|---------------|---------------|-----------------------------------------| | `cors` | `string` | no | | Container name for CORS configurations. | + +# `contracts` section + +```yaml +contracts: + container: + name: container.frostfs +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|------------------|----------|---------------|---------------------|----------------------------------------------| +| `container.name` | `string` | no | `container.frostfs` | Container contract hash (LE) or name in NNS. | diff --git a/go.mod b/go.mod index c065b57..6082ef6 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.23 require ( + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc @@ -33,7 +34,6 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect - git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e // indirect git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect diff --git a/internal/handler/container.go b/internal/handler/container.go new file mode 100644 index 0000000..3c7bec8 --- /dev/null +++ b/internal/handler/container.go @@ -0,0 +1,42 @@ +package handler + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "go.uber.org/zap" +) + +func (h *Handler) containerInfo(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) { + info := &data.BucketInfo{ + CID: cnrID, + Name: cnrID.EncodeToString(), + } + res, err := h.cnrContract.GetContainerByID(cnrID) + if err != nil { + return nil, fmt.Errorf("get frostfs container: %w", err) + } + + cnr := *res + + if domain := container.ReadDomain(cnr); domain.Name() != "" { + info.Name = domain.Name() + info.Zone = domain.Zone() + } + info.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(cnr) + info.PlacementPolicy = cnr.PlacementPolicy() + + if err = h.cache.Put(info); err != nil { + h.reqLogger(ctx).Warn(logs.CouldntPutBucketIntoCache, + zap.String("bucket name", info.Name), + zap.Stringer("cid", info.CID), + zap.Error(err), + logs.TagField(logs.TagDatapath)) + } + + return info, nil +} diff --git a/internal/handler/frostfs_mock.go b/internal/handler/frostfs_mock.go index 7d72ad9..540697f 100644 --- a/internal/handler/frostfs_mock.go +++ b/internal/handler/frostfs_mock.go @@ -233,6 +233,16 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) (Res return &resObjectSearchMock{res: res}, nil } +func (t *TestFrostFS) GetContainerByID(cid cid.ID) (*container.Container, error) { + for k, v := range t.containers { + if k == cid.EncodeToString() { + return v, nil + } + } + + return nil, fmt.Errorf("container does not exist %s", cid) +} + func (t *TestFrostFS) InitMultiObjectReader(context.Context, PrmInitMultiObjectReader) (io.Reader, error) { return nil, nil } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4d1dc31..2efd71d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -167,12 +167,18 @@ type ContainerResolver interface { Resolve(ctx context.Context, zone, name string) (*cid.ID, error) } +type ContainerContract interface { + // GetContainerByID reads a container from contract by ID. + GetContainerByID(cid.ID) (*container.Container, error) +} + type Handler struct { log *zap.Logger frostfs FrostFS ownerID *user.ID config Config containerResolver ContainerResolver + cnrContract ContainerContract tree *tree.Tree cache *cache.BucketCache workerPool *ants.Pool @@ -190,7 +196,7 @@ type AppParams struct { CORSCache *cache.CORSCache } -func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { +func New(params *AppParams, config Config, tree *tree.Tree, rpcCli ContainerContract, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, @@ -202,6 +208,7 @@ func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Poo workerPool: workerPool, corsCnrID: params.CORSCnrID, corsCache: params.CORSCache, + cnrContract: rpcCli, } } @@ -308,43 +315,7 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string) (*dat return nil, fmt.Errorf("resolve container: %w", err) } - bktInfo, err := h.readContainer(ctx, *cnrID) - if err != nil { - return nil, fmt.Errorf("read container: %w", err) - } - - if err = h.cache.Put(bktInfo); err != nil { - h.reqLogger(ctx).Warn(logs.CouldntPutBucketIntoCache, - zap.String("bucket name", bktInfo.Name), - zap.Stringer("bucket cid", bktInfo.CID), - zap.Error(err), - logs.TagField(logs.TagDatapath)) - } - - return bktInfo, nil -} - -func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) { - prm := PrmContainer{ContainerID: cnrID} - res, err := h.frostfs.Container(ctx, prm) - if err != nil { - return nil, fmt.Errorf("get frostfs container '%s': %w", cnrID.String(), err) - } - - bktInfo := &data.BucketInfo{ - CID: cnrID, - Name: cnrID.EncodeToString(), - } - - if domain := container.ReadDomain(*res); domain.Name() != "" { - bktInfo.Name = domain.Name() - bktInfo.Zone = domain.Zone() - } - - bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(*res) - bktInfo.PlacementPolicy = res.PlacementPolicy() - - return bktInfo, err + return h.containerInfo(ctx, *cnrID) } type ListFunc func(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 622940e..6c715fe 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -156,7 +156,7 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { if err != nil { return nil, err } - handler := New(params, cfgMock, tree.NewTree(treeMock, logger), workerPool) + handler := New(params, cfgMock, tree.NewTree(treeMock, logger), testFrostFS, workerPool) return &handlerContext{ key: key, diff --git a/internal/logs/logs.go b/internal/logs/logs.go index e7d118f..86921dd 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -73,6 +73,9 @@ const ( FailedToReadIndexPageTemplate = "failed to read index page template" SetCustomIndexPageTemplate = "set custom index page template" CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info" + InitRPCClientFailed = "init rpc client faileds" + InitContainerContractFailed = "init container contract failed" + FailedToResolveContractHash = "failed to resolve contract hash" ) // Log messages with the "datapath" tag. @@ -107,9 +110,7 @@ const ( IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" FailedToGetBucketInfo = "could not get bucket info" FailedToSubmitTaskToPool = "failed to submit task to pool" - ObjectWasDeleted = "object was deleted" IndexWasDeleted = "index was deleted" - FailedToGetLatestVersionOfObject = "failed to get latest version of object" FailedToGetLatestVersionOfIndexObject = "failed to get latest version of index object" FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists" FailedToListObjects = "failed to list objects" @@ -121,7 +122,6 @@ const ( FailedToGetObjectPayload = "failed to get object payload" FailedToFindObjectByAttribute = "failed to get find object by attribute" FailedToUnescapePath = "failed to unescape path" - InvalidOIDParam = "invalid oid param" CouldNotGetCORSConfiguration = "could not get cors configuration" EmptyOriginRequestHeader = "empty Origin request header" EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header" diff --git a/internal/service/contracts/container/client.go b/internal/service/contracts/container/client.go new file mode 100644 index 0000000..09455be --- /dev/null +++ b/internal/service/contracts/container/client.go @@ -0,0 +1,73 @@ +package container + +import ( + "fmt" + "strings" + + containercontract "git.frostfs.info/TrueCloudLab/frostfs-contract/container" + containerclient "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/container" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/rpcclient" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/wallet" +) + +type Client struct { + contract *containerclient.Contract +} + +type Config struct { + ContractHash util.Uint160 + Key *keys.PrivateKey + RPCClient *rpcclient.Client +} + +func New(cfg Config) (*Client, error) { + var err error + key := cfg.Key + if key == nil { + if key, err = keys.NewPrivateKey(); err != nil { + return nil, fmt.Errorf("generate anon private key for container contract: %w", err) + } + } + acc := wallet.NewAccountFromPrivateKey(key) + + act, err := actor.NewSimple(cfg.RPCClient, acc) + if err != nil { + return nil, fmt.Errorf("create new actor: %w", err) + } + + return &Client{ + contract: containerclient.New(act, cfg.ContractHash), + }, nil +} + +func (c *Client) GetContainerByID(cnrID cid.ID) (*container.Container, error) { + items, err := c.contract.Get(cnrID[:]) + if err != nil { + if strings.Contains(err.Error(), containercontract.NotFoundError) { + return nil, fmt.Errorf("%w: %s", handler.ErrContainerNotFound, err) + } + return nil, err + } + + if len(items) != 4 { + return nil, fmt.Errorf("unexpected container stack item count: %d", len(items)) + } + + cnrBytes, err := items[0].TryBytes() + if err != nil { + return nil, fmt.Errorf("could not get byte array of container: %w", err) + } + + var cnr container.Container + if err = cnr.Unmarshal(cnrBytes); err != nil { + return nil, fmt.Errorf("can't unmarshal container: %w", err) + } + + return &cnr, nil +} diff --git a/internal/service/contracts/util/util.go b/internal/service/contracts/util/util.go new file mode 100644 index 0000000..444504b --- /dev/null +++ b/internal/service/contracts/util/util.go @@ -0,0 +1,34 @@ +package util + +import ( + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +// ResolveContractHash determine contract hash by resolving NNS name. +func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error) { + if hash, err := util.Uint160DecodeStringLE(contractHash); err == nil { + return hash, nil + } + + splitName := strings.Split(contractHash, ".") + if len(splitName) != 2 { + return util.Uint160{}, fmt.Errorf("invalid contract name: '%s'", contractHash) + } + + var domain container.Domain + domain.SetName(splitName[0]) + domain.SetZone(splitName[1]) + + var nns ns.NNS + if err := nns.Dial(rpcAddress); err != nil { + return util.Uint160{}, fmt.Errorf("dial nns %s: %w", rpcAddress, err) + } + defer nns.Close() + + return nns.ResolveContractHash(domain) +}
FilenameOID Size Created Download
- ⮐.. + ⮐..
- ⮐.. + ⮐..
{{if .IsDir}} 🗀 - + {{.FileName}}/ {{else}} 🗎 - + {{.FileName}} {{end}} {{.OID}} {{if not .IsDir}}{{ formatSize .Size }}{{end}}{{if not .IsDir}}{{ formatTimestamp .Created }}{{end}}{{ .Created }} - {{ if not .IsDir }} - + {{ if .OID }} + Link {{ end }} diff --git a/tree/tree.go b/tree/tree.go index 162f41f..40209a5 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -30,6 +30,11 @@ type ( Meta map[string]string } + multiSystemNode struct { + // the first element is latest + nodes []*treeNode + } + GetNodesParams struct { CnrID cid.ID BktInfo *data.BucketInfo @@ -50,18 +55,19 @@ var ( ) const ( - FileNameKey = "FileName" -) + FileNameKey = "FileName" + settingsFileName = "bucket-settings" -const ( - oidKV = "OID" + oidKV = "OID" + uploadIDKV = "UploadId" + sizeKV = "Size" // keys for delete marker nodes. isDeleteMarkerKV = "IsDeleteMarker" - sizeKV = "Size" // versionTree -- ID of a tree with object versions. versionTree = "version" + systemTree = "system" separator = "/" ) @@ -135,6 +141,45 @@ func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { return version } +func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) { + var ( + err error + index int + maxTimestamp uint64 + ) + + if len(nodes) == 0 { + return nil, errors.New("multi node must have at least one node") + } + + treeNodes := make([]*treeNode, len(nodes)) + + for i, node := range nodes { + if treeNodes[i], err = newTreeNode(node); err != nil { + return nil, fmt.Errorf("parse system node response: %w", err) + } + + if timestamp := getMaxTimestamp(node); timestamp > maxTimestamp { + index = i + maxTimestamp = timestamp + } + } + + treeNodes[0], treeNodes[index] = treeNodes[index], treeNodes[0] + + return &multiSystemNode{ + nodes: treeNodes, + }, nil +} + +func (m *multiSystemNode) Latest() *treeNode { + return m.nodes[0] +} + +func (m *multiSystemNode) Old() []*treeNode { + return m.nodes[1:] +} + func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { nodes, err := c.GetVersions(ctx, cnrID, objectName) if err != nil { @@ -165,6 +210,55 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string return c.service.GetNodes(ctx, p) } +func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error { + _, err := c.getSystemNode(ctx, bktInfo, settingsFileName) + if err != nil { + return err + } + + return nil +} + +func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) { + p := &GetNodesParams{ + CnrID: bktInfo.CID, + BktInfo: bktInfo, + TreeID: systemTree, + Path: []string{name}, + LatestOnly: false, + AllAttrs: true, + } + nodes, err := c.service.GetNodes(ctx, p) + if err != nil { + return nil, err + } + + nodes = filterMultipartNodes(nodes) + + if len(nodes) == 0 { + return nil, ErrNodeNotFound + } + + return newMultiNode(nodes) +} + +func filterMultipartNodes(nodes []NodeResponse) []NodeResponse { + res := make([]NodeResponse, 0, len(nodes)) + +LOOP: + for _, node := range nodes { + for _, meta := range node.GetMeta() { + if meta.GetKey() == uploadIDKV { + continue LOOP + } + } + + res = append(res, node) + } + + return res +} + func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { var ( maxCreationTime uint64 From a2f8cb673539c350e2b4c147c1510078fa7bd59e Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 20 Nov 2024 11:09:31 +0300 Subject: [PATCH 25/86] Release v0.31.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++----- VERSION | 2 +- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0990b51..dc422d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,51 @@ This document outlines major changes between releases. ## [Unreleased] +## [0.31.0] - Rongbuk - 2024-11-20 + +### Fixed +- Docker warnings during image build (#126) +- `trace_id` parameter in logs (#148) +- SIGHUP support for `tracing.enabled` config parameter (#157) + ### Added -- Support percent-encoding for GET queries (#134) -- Add `trace_id` to logs (#148) -- Add `cors` config params (#158) +- Vulnerability report document (#123) +- Root CA configuration for tracing (#139) +- Log sampling policy configuration (#147) +- Index page support for buckets and containers (#137, #151) +- CORS support (#158) +- Source IP binding configuration for FrostFS requests (#160) +- Tracing attributes (#164) ### Changed -- Update go version to 1.22 (#132) +- Updated Go version to 1.22 (#132) + +### Removed +- Duplicated NNS Resolver code (#129) + +## [0.30.3] - 2024-10-18 + +### Fixed +- Get response on S3 multipart object (#142) + +### Added +- Support percent-encoding for GET queries (#134) + +### Changed +- Split `FrostFS` interface into separate read methods (#127) + +## [0.30.2] - 2024-09-03 + +### Added +- Fuzzing tests (#135) + +## [0.30.1] - 2024-08-20 + +### Fixed +- Error counting in pool component before connection switch (#131) + +### Added +- Log of endpoint address during tree pool errors (#131) ## [0.30.0] - Kangshung - 2024-07-22 @@ -128,4 +166,8 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.0...v0.28.1 [0.29.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.1...v0.29.0 [0.30.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.29.0...v0.30.0 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.0...master +[0.30.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.0...v0.30.1 +[0.30.2]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.1...v0.30.2 +[0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.2...v0.30.3 +[0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.3...v0.31.0 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...master \ No newline at end of file diff --git a/VERSION b/VERSION index 9388ecb..7021025 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.30.0 +v0.31.0 From e81f01c2abfb0958da757065c3d795a7b11e14bb Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Sun, 24 Nov 2024 13:32:40 +0300 Subject: [PATCH 26/86] [#150] Add dropped logs metric Signed-off-by: Pavel Pogodaev --- cmd/http-gw/app.go | 92 ++++++++++++++++----------------- cmd/http-gw/integration_test.go | 4 +- cmd/http-gw/main.go | 3 +- cmd/http-gw/settings.go | 52 +++++++++++-------- metrics/desc.go | 18 +++++++ metrics/metrics.go | 69 ++++++++++++++++++------- 6 files changed, 147 insertions(+), 91 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 0dd53a6..6ac9b1c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -50,35 +50,38 @@ import ( type ( app struct { - ctx context.Context - log *zap.Logger - logLevel zap.AtomicLevel - pool *pool.Pool - treePool *treepool.Pool - key *keys.PrivateKey - owner *user.ID - cfg *viper.Viper - webServer *fasthttp.Server - webDone chan struct{} - resolver *resolver.ContainerResolver - metrics *gateMetrics - services []*metrics.Service - settings *appSettings + ctx context.Context + log *zap.Logger + logLevel zap.AtomicLevel + pool *pool.Pool + treePool *treepool.Pool + key *keys.PrivateKey + owner *user.ID + cfg *viper.Viper + webServer *fasthttp.Server + webDone chan struct{} + resolver *resolver.ContainerResolver + metrics *gateMetrics + services []*metrics.Service + settings *appSettings + loggerSettings *loggerSettings servers []Server unbindServers []ServerInfo mu sync.RWMutex } + loggerSettings struct { + mu sync.RWMutex + appMetrics *metrics.GateMetrics + } + // App is an interface for the main gateway function. App interface { Wait() Serve() } - // Option is an application option. - Option func(a *app) - gateMetrics struct { logger *zap.Logger provider *metrics.GateMetrics @@ -119,37 +122,17 @@ type ( } ) -// WithLogger returns Option to set a specific logger. -func WithLogger(l *zap.Logger, lvl zap.AtomicLevel) Option { - return func(a *app) { - if l == nil { - return - } - a.log = l - a.logLevel = lvl - } -} +func newApp(ctx context.Context, v *viper.Viper) App { + logSettings := &loggerSettings{} + log := pickLogger(v, logSettings) -// WithConfig returns Option to use specific Viper configuration. -func WithConfig(c *viper.Viper) Option { - return func(a *app) { - if c == nil { - return - } - a.cfg = c - } -} - -func newApp(ctx context.Context, opt ...Option) App { a := &app{ - ctx: ctx, - log: zap.L(), - cfg: viper.GetViper(), - webServer: new(fasthttp.Server), - webDone: make(chan struct{}), - } - for i := range opt { - opt[i](a) + ctx: ctx, + log: log.logger, + cfg: v, + loggerSettings: logSettings, + webServer: new(fasthttp.Server), + webDone: make(chan struct{}), } a.initAppSettings() @@ -227,6 +210,22 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.corsMaxAge = corsMaxAge } +func (s *loggerSettings) DroppedLogsInc() { + s.mu.RLock() + defer s.mu.RUnlock() + + if s.appMetrics != nil { + s.appMetrics.DroppedLogsInc() + } +} + +func (s *loggerSettings) setMetrics(appMetrics *metrics.GateMetrics) { + s.mu.Lock() + defer s.mu.Unlock() + + s.appMetrics = appMetrics +} + func (s *appSettings) DefaultTimestamp() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -338,6 +337,7 @@ func (a *app) initMetrics() { gateMetricsProvider := metrics.NewGateMetrics(a.pool) a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled)) a.metrics.SetHealth(metrics.HealthStatusStarting) + a.loggerSettings.setMetrics(a.metrics.provider) } func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics { diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 79a2da5..1516f1d 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -34,7 +34,6 @@ import ( "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" - "go.uber.org/zap/zapcore" ) type putResponse struct { @@ -102,8 +101,7 @@ func runServer(pathToWallet string) (App, context.CancelFunc) { v.Set(cfgWalletPath, pathToWallet) v.Set(cfgWalletPassphrase, "") - l, lvl := newStdoutLogger(v, zapcore.DebugLevel) - application := newApp(cancelCtx, WithConfig(v), WithLogger(l, lvl)) + application := newApp(cancelCtx, v) go application.Serve() return application, cancel diff --git a/cmd/http-gw/main.go b/cmd/http-gw/main.go index ea9fbd7..fdd148c 100644 --- a/cmd/http-gw/main.go +++ b/cmd/http-gw/main.go @@ -9,9 +9,8 @@ import ( func main() { globalContext, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) v := settings() - logger, atomicLevel := pickLogger(v) - application := newApp(globalContext, WithLogger(logger, atomicLevel), WithConfig(v)) + application := newApp(globalContext, v) go application.Serve() application.Wait() } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 316c500..2298124 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -182,6 +182,11 @@ var ignore = map[string]struct{}{ cmdVersion: {}, } +type Logger struct { + logger *zap.Logger + lvl zap.AtomicLevel +} + func settings() *viper.Viper { v := viper.New() v.AutomaticEnv() @@ -418,7 +423,11 @@ func mergeConfig(v *viper.Viper, fileName string) error { return v.MergeConfig(cfgFile) } -func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { +type LoggerAppSettings interface { + DroppedLogsInc() +} + +func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger { lvl, err := getLogLevel(v) if err != nil { panic(err) @@ -428,9 +437,9 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { switch dest { case destinationStdout: - return newStdoutLogger(v, lvl) + return newStdoutLogger(v, lvl, settings) case destinationJournald: - return newJournaldLogger(v, lvl) + return newJournaldLogger(v, lvl, settings) default: panic(fmt.Sprintf("wrong destination for logger: %s", dest)) } @@ -447,18 +456,20 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { // Logger records a stack trace for all messages at or above fatal level. // // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. -func newStdoutLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { +func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger { stdout := zapcore.AddSync(os.Stderr) level := zap.NewAtomicLevelAt(lvl) consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level) - consoleOutCore = samplingEnabling(v, consoleOutCore) + consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, settings) - l := zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) - return l, level + return &Logger{ + logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), + lvl: level, + } } -func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { +func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger { level := zap.NewAtomicLevelAt(lvl) encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) @@ -470,11 +481,12 @@ func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.Atom zapjournald.SyslogPid(), }) - coreWithContext = samplingEnabling(v, coreWithContext) + coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, settings) - l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) - - return l, level + return &Logger{ + logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), + lvl: level, + } } func newLogEncoder() zapcore.Encoder { @@ -484,19 +496,17 @@ func newLogEncoder() zapcore.Encoder { return zapcore.NewConsoleEncoder(c) } -func samplingEnabling(v *viper.Viper, core zapcore.Core) zapcore.Core { - // Zap samples by logging the first cgfLoggerSamplingInitial entries with a given level - // and message within the specified time interval. - // In the above config, only the first cgfLoggerSamplingInitial log entries with the same level and message - // are recorded in cfgLoggerSamplingInterval interval. Every other log entry will be dropped within the interval since - // cfgLoggerSamplingThereafter is specified here. +func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, settings LoggerAppSettings) zapcore.Core { if v.GetBool(cfgLoggerSamplingEnabled) { - core = zapcore.NewSamplerWithOptions( - core, + core = zapcore.NewSamplerWithOptions(core, v.GetDuration(cfgLoggerSamplingInterval), v.GetInt(cfgLoggerSamplingInitial), v.GetInt(cfgLoggerSamplingThereafter), - ) + zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) { + if dec&zapcore.LogDropped > 0 { + settings.DroppedLogsInc() + } + })) } return core diff --git a/metrics/desc.go b/metrics/desc.go index e10050c..a00ab3e 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -76,6 +76,15 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"endpoint"}, }, }, + statisticSubsystem: { + droppedLogs: Description{ + Type: dto.MetricType_COUNTER, + Namespace: namespace, + Subsystem: statisticSubsystem, + Name: droppedLogs, + Help: "Dropped logs (by sampling) count", + }, + }, } type Description struct { @@ -148,3 +157,12 @@ func mustNewGaugeVec(description Description) *prometheus.GaugeVec { description.VariableLabels, ) } + +func mustNewCounter(description Description) prometheus.Counter { + if description.Type != dto.MetricType_COUNTER { + panic("invalid metric type") + } + return prometheus.NewCounter( + prometheus.CounterOpts(newOpts(description)), + ) +} diff --git a/metrics/metrics.go b/metrics/metrics.go index b516477..1c06868 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -10,15 +10,17 @@ import ( ) const ( - namespace = "frostfs_http_gw" - stateSubsystem = "state" - poolSubsystem = "pool" - serverSubsystem = "server" + namespace = "frostfs_http_gw" + stateSubsystem = "state" + poolSubsystem = "pool" + serverSubsystem = "server" + statisticSubsystem = "statistic" ) const ( healthMetric = "health" versionInfoMetric = "version_info" + droppedLogs = "dropped_logs" ) const ( @@ -30,21 +32,19 @@ const ( ) const ( - methodGetBalance = "get_balance" - methodPutContainer = "put_container" - methodGetContainer = "get_container" - methodListContainer = "list_container" - methodDeleteContainer = "delete_container" - methodGetContainerEacl = "get_container_eacl" - methodSetContainerEacl = "set_container_eacl" - methodEndpointInfo = "endpoint_info" - methodNetworkInfo = "network_info" - methodPutObject = "put_object" - methodDeleteObject = "delete_object" - methodGetObject = "get_object" - methodHeadObject = "head_object" - methodRangeObject = "range_object" - methodCreateSession = "create_session" + methodGetBalance = "get_balance" + methodPutContainer = "put_container" + methodGetContainer = "get_container" + methodListContainer = "list_container" + methodDeleteContainer = "delete_container" + methodEndpointInfo = "endpoint_info" + methodNetworkInfo = "network_info" + methodPutObject = "put_object" + methodDeleteObject = "delete_object" + methodGetObject = "get_object" + methodHeadObject = "head_object" + methodRangeObject = "range_object" + methodCreateSession = "create_session" ) // HealthStatus of the gate application. @@ -69,6 +69,7 @@ type GateMetrics struct { stateMetrics poolMetricsCollector serverMetrics + statisticMetrics } type stateMetrics struct { @@ -76,6 +77,10 @@ type stateMetrics struct { versionInfo *prometheus.GaugeVec } +type statisticMetrics struct { + droppedLogs prometheus.Counter +} + type poolMetricsCollector struct { scraper StatisticScraper overallErrors prometheus.Gauge @@ -96,10 +101,14 @@ func NewGateMetrics(p StatisticScraper) *GateMetrics { serverMetric := newServerMetrics() serverMetric.register() + statsMetric := newStatisticMetrics() + statsMetric.register() + return &GateMetrics{ stateMetrics: *stateMetric, poolMetricsCollector: *poolMetric, serverMetrics: *serverMetric, + statisticMetrics: *statsMetric, } } @@ -107,6 +116,7 @@ func (g *GateMetrics) Unregister() { g.stateMetrics.unregister() prometheus.Unregister(&g.poolMetricsCollector) g.serverMetrics.unregister() + g.statisticMetrics.unregister() } func newStateMetrics() *stateMetrics { @@ -116,6 +126,20 @@ func newStateMetrics() *stateMetrics { } } +func newStatisticMetrics() *statisticMetrics { + return &statisticMetrics{ + droppedLogs: mustNewCounter(appMetricsDesc[statisticSubsystem][droppedLogs]), + } +} + +func (s *statisticMetrics) register() { + prometheus.MustRegister(s.droppedLogs) +} + +func (s *statisticMetrics) unregister() { + prometheus.Unregister(s.droppedLogs) +} + func (m stateMetrics) register() { prometheus.MustRegister(m.healthCheck) prometheus.MustRegister(m.versionInfo) @@ -134,6 +158,13 @@ func (m stateMetrics) SetVersion(ver string) { m.versionInfo.WithLabelValues(ver).Set(1) } +func (s *statisticMetrics) DroppedLogsInc() { + if s == nil { + return + } + s.droppedLogs.Inc() +} + func newPoolMetricsCollector(p StatisticScraper) *poolMetricsCollector { return &poolMetricsCollector{ scraper: p, From b9e44c603dc2a4605de5119238e01aeb2086f9a7 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 9 Dec 2024 14:45:24 +0300 Subject: [PATCH 27/86] [#178] Update frostfs-sdk-go with new tree service client Add tree service's GetBucketSettings to use them to check for protocol to use (S3 or native). Also add mock implementations for this and GetLatestVersion methods. Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 2 +- cmd/http-gw/integration_test.go | 2 +- go.mod | 3 +-- go.sum | 6 ++---- internal/service/frostfs/pool_wrapper.go | 22 +++++++++++----------- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 6ac9b1c..53c001c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -16,7 +16,6 @@ import ( "syscall" "time" - v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" @@ -31,6 +30,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 1516f1d..dd4d60f 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -17,7 +17,7 @@ import ( "testing" "time" - containerv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" + containerv2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" diff --git a/go.mod b/go.mod index a2f41d8..3dd27b8 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,8 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.22 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 diff --git a/go.sum b/go.sum index a7a5be4..7f43c86 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 h1:ivcdxQeQDnx4srF2ezoaeVlF0FAycSAztwfIUJnUI4s= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 h1:f7jan6eBDN88DKnKj8GKyWpfjBbSzjDALcDejYKRgCs= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3/go.mod h1:3txOjFJ8M/JFs01h7xOrnQHVn6hZgDNA16ivyUlu1iU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d h1:FpXI+mOrmJk3t2MKQFZuhLjCHDyDeo5rtP1WXl7gUWc= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= diff --git a/internal/service/frostfs/pool_wrapper.go b/internal/service/frostfs/pool_wrapper.go index b978d73..f6be05f 100644 --- a/internal/service/frostfs/pool_wrapper.go +++ b/internal/service/frostfs/pool_wrapper.go @@ -9,20 +9,20 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" + apitree "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/tree" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" - grpcService "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree/service" ) type GetNodeByPathResponseInfoWrapper struct { - response *grpcService.GetNodeByPathResponse_Info + response *apitree.GetNodeByPathResponseInfo } func (n GetNodeByPathResponseInfoWrapper) GetNodeID() []uint64 { - return []uint64{n.response.GetNodeId()} + return []uint64{n.response.GetNodeID()} } func (n GetNodeByPathResponseInfoWrapper) GetParentID() []uint64 { - return []uint64{n.response.GetParentId()} + return []uint64{n.response.GetParentID()} } func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() []uint64 { @@ -30,8 +30,8 @@ func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() []uint64 { } func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { - res := make([]tree.Meta, len(n.response.Meta)) - for i, value := range n.response.Meta { + res := make([]tree.Meta, len(n.response.GetMeta())) + for i, value := range n.response.GetMeta() { res[i] = value } return res @@ -133,15 +133,15 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, } type GetSubTreeResponseBodyWrapper struct { - response *grpcService.GetSubTreeResponse_Body + response *apitree.GetSubTreeResponseBody } func (n GetSubTreeResponseBodyWrapper) GetNodeID() []uint64 { - return n.response.GetNodeId() + return n.response.GetNodeID() } func (n GetSubTreeResponseBodyWrapper) GetParentID() []uint64 { - resp := n.response.GetParentId() + resp := n.response.GetParentID() if resp == nil { // storage sends nil that should be interpreted as []uint64{0} // due to protobuf compatibility, see 'GetSubTree' function @@ -155,8 +155,8 @@ func (n GetSubTreeResponseBodyWrapper) GetTimestamp() []uint64 { } func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { - res := make([]tree.Meta, len(n.response.Meta)) - for i, value := range n.response.Meta { + res := make([]tree.Meta, len(n.response.GetMeta())) + for i, value := range n.response.GetMeta() { res[i] = value } return res From bbc7c7367d3146e8e1bbc13d60f08022b0d6fcc1 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 10 Dec 2024 15:42:13 +0300 Subject: [PATCH 28/86] [#179] Refine CODEOWNERS settings Signed-off-by: Vitaliy Potyarkin --- CODEOWNERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 43df11e..de5e48e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,3 @@ -.* @alexvanin @dkirillov +.* @TrueCloudLab/storage-services-developers @TrueCloudLab/storage-services-committers +.forgejo/.* @potyarkin +Makefile @potyarkin From dc100f03a6a164b9c49fee87bf663670c114ca5d Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 12 Dec 2024 09:28:22 +0300 Subject: [PATCH 29/86] [#174] Add fallback path to search Fallback path to search is needed because some software may keep FileName attribute and ignore FilePath attribute during file upload. Therefore, if this feature is enabled under certain conditions (for more information, see gate-configuration.md) a search will be performed for the FileName attribute. Signed-off-by: Roman Loginov --- cmd/http-gw/app.go | 39 ++++--- cmd/http-gw/settings.go | 3 + config/config.env | 5 +- config/config.yaml | 4 + docs/gate-configuration.md | 15 ++- internal/handler/browse.go | 1 + internal/handler/handler.go | 52 +++++++--- internal/handler/handler_test.go | 171 ++++++++++++++++++++++++++++++- internal/logs/logs.go | 1 + 9 files changed, 256 insertions(+), 35 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 53c001c..9adaec2 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -95,21 +95,22 @@ type ( dialerSource *internalnet.DialerSource workerPoolSize int - mu sync.RWMutex - defaultTimestamp bool - zipCompression bool - clientCut bool - returnIndexPage bool - indexPageTemplate string - bufferMaxSizeForPut uint64 - namespaceHeader string - defaultNamespaces []string - corsAllowOrigin string - corsAllowMethods []string - corsAllowHeaders []string - corsExposeHeaders []string - corsAllowCredentials bool - corsMaxAge int + mu sync.RWMutex + defaultTimestamp bool + zipCompression bool + clientCut bool + returnIndexPage bool + indexPageTemplate string + bufferMaxSizeForPut uint64 + namespaceHeader string + defaultNamespaces []string + corsAllowOrigin string + corsAllowMethods []string + corsAllowHeaders []string + corsExposeHeaders []string + corsAllowCredentials bool + corsMaxAge int + enableFilepathFallback bool } CORS struct { @@ -189,6 +190,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { corsExposeHeaders := v.GetStringSlice(cfgCORSExposeHeaders) corsAllowCredentials := v.GetBool(cfgCORSAllowCredentials) corsMaxAge := fetchCORSMaxAge(v) + enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback) s.mu.Lock() defer s.mu.Unlock() @@ -208,6 +210,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.corsExposeHeaders = corsExposeHeaders s.corsAllowCredentials = corsAllowCredentials s.corsMaxAge = corsMaxAge + s.enableFilepathFallback = enableFilepathFallback } func (s *loggerSettings) DroppedLogsInc() { @@ -305,6 +308,12 @@ func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) return ns + ".ns", false } +func (s *appSettings) EnableFilepathFallback() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.enableFilepathFallback +} + func (a *app) initResolver() { var err error a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 2298124..62ef83e 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -164,6 +164,9 @@ const ( cfgMultinetFallbackDelay = "multinet.fallback_delay" cfgMultinetSubnets = "multinet.subnets" + // Feature. + cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" + // Command line args. cmdHelp = "help" cmdVersion = "version" diff --git a/config/config.env b/config/config.env index fd51392..2822357 100644 --- a/config/config.env +++ b/config/config.env @@ -158,4 +158,7 @@ HTTP_GW_WORKER_POOL_SIZE=1000 # Enable index page support HTTP_GW_INDEX_PAGE_ENABLED=false # Index page template path -HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl \ No newline at end of file +HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl + +# Enable using fallback path to search for a object by attribute +HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false diff --git a/config/config.yaml b/config/config.yaml index ef5c529..6296bd9 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -172,3 +172,7 @@ multinet: source_ips: - 1.2.3.4 - 1.2.3.5 + +features: + # Enable using fallback path to search for a object by attribute + enable_filepath_fallback: false diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index c6cb617..7476f5d 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -59,7 +59,7 @@ $ cat http.log | `resolve_bucket` | [Bucket name resolving configuration](#resolve_bucket-section) | | `index_page` | [Index page configuration](#index_page-section) | | `multinet` | [Multinet configuration](#multinet-section) | - +| `features` | [Features configuration](#features-section) | # General section @@ -457,3 +457,16 @@ multinet: |--------------|------------|---------------|---------------|----------------------------------------------------------------------| | `mask` | `string` | yes | | Destination subnet. | | `source_ips` | `[]string` | yes | | Array of source IP addresses to use when dialing destination subnet. | + +# `features` section + +Contains parameters for enabling features. + +```yaml +features: + enable_filepath_fallback: true +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +| ----------------------------------- | ------ | ------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. | diff --git a/internal/handler/browse.go b/internal/handler/browse.go index b24a569..c54ab76 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -26,6 +26,7 @@ const ( attrOID = "OID" attrCreated = "Created" attrFileName = "FileName" + attrFilePath = "FilePath" attrSize = "Size" ) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9ed7f99..1839bf0 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -35,6 +35,7 @@ type Config interface { IndexPageTemplate() string BufferMaxSizeForPut() uint64 NamespaceHeader() string + EnableFilepathFallback() bool } // PrmContainer groups parameters of FrostFS.Container operation. @@ -291,35 +292,58 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re return } - res, err := h.search(ctx, bktInfo.CID, key, val, object.MatchStringEqual) + objID, err := h.findObjectByAttribute(ctx, log, bktInfo.CID, key, val) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) - response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) + if errors.Is(err, io.EOF) { + response.Error(c, err.Error(), fasthttp.StatusNotFound) + return + } + + response.Error(c, err.Error(), fasthttp.StatusBadRequest) return } + var addrObj oid.Address + addrObj.SetContainer(bktInfo.CID) + addrObj.SetObject(objID) + + f(ctx, *h.newRequest(c, log), addrObj) +} + +func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { + res, err := h.search(ctx, cnrID, attrKey, attrVal, object.MatchStringEqual) + if err != nil { + log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + return oid.ID{}, fmt.Errorf("could not search for objects: %w", err) + } defer res.Close() buf := make([]oid.ID, 1) n, err := res.Read(buf) if n == 0 { - if errors.Is(err, io.EOF) { + switch { + case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): + log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName) + return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) + case errors.Is(err, io.EOF): log.Error(logs.ObjectNotFound, zap.Error(err)) - response.Error(c, "object not found", fasthttp.StatusNotFound) - return + return oid.ID{}, fmt.Errorf("object not found: %w", err) + default: + log.Error(logs.ReadObjectListFailed, zap.Error(err)) + return oid.ID{}, fmt.Errorf("read object list failed: %w", err) } - - log.Error(logs.ReadObjectListFailed, zap.Error(err)) - response.Error(c, "read object list failed: "+err.Error(), fasthttp.StatusBadRequest) - return } - var addrObj oid.Address - addrObj.SetContainer(bktInfo.CID) - addrObj.SetObject(buf[0]) + return buf[0], nil +} - f(ctx, *h.newRequest(c, log), addrObj) +func (h *Handler) needSearchByFileName(key, val string) bool { + if key != attrFilePath || !h.config.EnableFilepathFallback() { + return false + } + + return strings.HasPrefix(val, "/") && strings.Count(val, "/") == 1 || !strings.Contains(val, "/") } // resolveContainer decode container id, if it's not a valid container id diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 34668a5..86364d9 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -44,6 +44,7 @@ func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, [ } type configMock struct { + additionalSearch bool } func (c *configMock) DefaultTimestamp() bool { @@ -78,6 +79,10 @@ func (c *configMock) NamespaceHeader() string { return "" } +func (c *configMock) EnableFilepathFallback() bool { + return c.additionalSearch +} + type handlerContext struct { key *keys.PrivateKey owner user.ID @@ -199,10 +204,8 @@ func TestBasic(t *testing.T) { require.NoError(t, err) obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] - attr := object.NewAttribute() - attr.SetKey(object.AttributeFilePath) - attr.SetValue(objFileName) - obj.SetAttributes(append(obj.Attributes(), *attr)...) + attr := prepareObjectAttributes(object.AttributeFilePath, objFileName) + obj.SetAttributes(append(obj.Attributes(), attr)...) t.Run("get", func(t *testing.T) { r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID) @@ -251,6 +254,159 @@ func TestBasic(t *testing.T) { }) } +func TestFindObjectByAttribute(t *testing.T) { + hc, err := prepareHandlerContext() + require.NoError(t, err) + hc.cfg.additionalSearch = true + + bktName := "bucket" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + content := "hello" + r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content) + require.NoError(t, err) + + hc.Handler().Upload(r) + require.Equal(t, r.Response.StatusCode(), http.StatusOK) + + var putRes putResponse + err = json.Unmarshal(r.Response.Body(), &putRes) + require.NoError(t, err) + + testAttrVal1 := "test-attr-val1" + testAttrVal2 := "test-attr-val2" + testAttrVal3 := "test-attr-val3" + + for _, tc := range []struct { + name string + firstAttr object.Attribute + secondAttr object.Attribute + reqAttrKey string + reqAttrValue string + err string + additionalSearch bool + }{ + { + name: "success search by FileName", + firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), + secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), + reqAttrKey: attrFileName, + reqAttrValue: testAttrVal2, + additionalSearch: false, + }, + { + name: "failed search by FileName", + firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), + secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), + reqAttrKey: attrFileName, + reqAttrValue: testAttrVal3, + err: "not found", + additionalSearch: false, + }, + { + name: "success search by FilePath (with additional search)", + firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), + secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), + reqAttrKey: attrFilePath, + reqAttrValue: testAttrVal2, + additionalSearch: true, + }, + { + name: "failed by FilePath (with additional search)", + firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), + secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), + reqAttrKey: attrFilePath, + reqAttrValue: testAttrVal3, + err: "not found", + additionalSearch: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] + obj.SetAttributes(tc.firstAttr, tc.secondAttr) + hc.cfg.additionalSearch = tc.additionalSearch + + objID, err := hc.Handler().findObjectByAttribute(ctx, hc.Handler().log, cnrID, tc.reqAttrKey, tc.reqAttrValue) + if tc.err != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tc.err) + return + } + + require.NoError(t, err) + require.Equal(t, putRes.ObjectID, objID.EncodeToString()) + }) + } +} + +func TestNeedSearchByFileName(t *testing.T) { + hc, err := prepareHandlerContext() + require.NoError(t, err) + + for _, tc := range []struct { + name string + attrKey string + attrVal string + additionalSearch bool + expected bool + }{ + { + name: "need search - not contains slash", + attrKey: attrFilePath, + attrVal: "cat.png", + additionalSearch: true, + expected: true, + }, + { + name: "need search - single lead slash", + attrKey: attrFilePath, + attrVal: "/cat.png", + additionalSearch: true, + expected: true, + }, + { + name: "don't need search - single slash but not lead", + attrKey: attrFilePath, + attrVal: "cats/cat.png", + additionalSearch: true, + expected: false, + }, + { + name: "don't need search - more one slash", + attrKey: attrFilePath, + attrVal: "/cats/cat.png", + additionalSearch: true, + expected: false, + }, + { + name: "don't need search - incorrect attribute key", + attrKey: attrFileName, + attrVal: "cat.png", + additionalSearch: true, + expected: false, + }, + { + name: "don't need search - additional search disabled", + attrKey: attrFilePath, + attrVal: "cat.png", + additionalSearch: false, + expected: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + hc.cfg.additionalSearch = tc.additionalSearch + + res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal) + require.Equal(t, tc.expected, res) + }) + } +} + func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) @@ -283,6 +439,13 @@ func prepareGetZipped(ctx context.Context, bucket, prefix string) *fasthttp.Requ return r } +func prepareObjectAttributes(attrKey, attrValue string) object.Attribute { + attr := object.NewAttribute() + attr.SetKey(attrKey) + attr.SetValue(attrValue) + return *attr +} + const ( keyAttr = "User-Attribute" valAttr = "user value" diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 4dfa21f..8c8b336 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -87,4 +87,5 @@ const ( MultinetDialFail = "multinet dial failed" FailedToLoadMultinetConfig = "failed to load multinet config" MultinetConfigWontBeUpdated = "multinet config won't be updated" + ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" ) From 1be92fa4befc8f95bf94f5aff39e05c87c57d7f8 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 2 Dec 2024 11:45:30 +0300 Subject: [PATCH 30/86] [#166] Fix getting s3 object with the FrostFS OID name Prioritize getting s3 object with the key, which equals to valid FrostFS OID, rather than getting non-existent object with OID via native protocol for GET and HEAD requests Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 4 +- internal/data/{bucket.go => info.go} | 0 internal/{api => data}/tree.go | 17 ++- internal/handler/browse.go | 2 +- internal/handler/download.go | 49 +++++++-- internal/handler/handler.go | 104 +++++------------- internal/handler/handler_test.go | 33 ++++-- internal/handler/head.go | 34 +++++- internal/handler/utils.go | 11 +- internal/{api => }/layer/tree_service.go | 6 +- internal/logs/logs.go | 5 + .../{pool_wrapper.go => tree_pool_wrapper.go} | 0 tree/tree.go | 52 ++++++--- 13 files changed, 178 insertions(+), 139 deletions(-) rename internal/data/{bucket.go => info.go} (100%) rename internal/{api => data}/tree.go (61%) rename internal/{api => }/layer/tree_service.go (64%) rename internal/service/frostfs/{pool_wrapper.go => tree_pool_wrapper.go} (100%) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 9adaec2..23a752a 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -508,10 +508,10 @@ func (a *app) Serve() { close(a.webDone) }() - handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) + handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) // Configure router. - a.configureRouter(handler) + a.configureRouter(handle) a.startServices() a.initServers(a.ctx) diff --git a/internal/data/bucket.go b/internal/data/info.go similarity index 100% rename from internal/data/bucket.go rename to internal/data/info.go diff --git a/internal/api/tree.go b/internal/data/tree.go similarity index 61% rename from internal/api/tree.go rename to internal/data/tree.go index 5b1d608..fcf8add 100644 --- a/internal/api/tree.go +++ b/internal/data/tree.go @@ -1,4 +1,4 @@ -package api +package data import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -7,12 +7,21 @@ import ( // NodeVersion represent node from tree service. type NodeVersion struct { BaseNodeVersion - DeleteMarker bool - IsPrefixNode bool } // BaseNodeVersion is minimal node info from tree service. // Basically used for "system" object. type BaseNodeVersion struct { - OID oid.ID + ID uint64 + OID oid.ID + IsDeleteMarker bool +} + +type NodeInfo struct { + Meta []NodeMeta +} + +type NodeMeta interface { + GetKey() string + GetValue() []byte } diff --git a/internal/handler/browse.go b/internal/handler/browse.go index c54ab76..2938b5c 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -170,7 +170,7 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn objects: make([]ResponseObject, 0, len(nodes)), } for _, node := range nodes { - meta := node.GetMeta() + meta := node.Meta if meta == nil { continue } diff --git a/internal/handler/download.go b/internal/handler/download.go index cd4e55a..de27fa3 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -4,12 +4,14 @@ import ( "archive/zip" "bufio" "context" + "errors" "fmt" "io" "net/http" "net/url" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -23,21 +25,46 @@ import ( // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { - oidURLParam := c.UserValue("oid").(string) - downloadQueryParam := c.QueryArgs().GetBool("download") + cidParam := c.UserValue("cid").(string) + oidParam := c.UserValue("oid").(string) + downloadParam := c.QueryArgs().GetBool("download") - switch { - case isObjectID(oidURLParam): - h.byNativeAddress(c, h.receiveFile) - case !isContainerRoot(oidURLParam) && (downloadQueryParam || !isDir(oidURLParam)): - h.byS3Path(c, h.receiveFile) - default: - h.browseIndex(c) + ctx := utils.GetContextFromRequest(c) + log := utils.GetReqLogOrDefault(ctx, h.log).With( + zap.String("cid", cidParam), + zap.String("oid", oidParam), + ) + + bktInfo, err := h.getBucketInfo(ctx, cidParam, log) + if err != nil { + logAndSendBucketError(c, log, err) + return + } + + checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) + if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + logAndSendBucketError(c, log, checkS3Err) + return + } + + req := h.newRequest(c, log) + + var objID oid.ID + if checkS3Err == nil && shouldDownload(oidParam, downloadParam) { + h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile) + } else if err = objID.DecodeString(oidParam); err == nil { + h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile) + } else { + h.browseIndex(c, checkS3Err != nil) } } -func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { - return &request{ +func shouldDownload(oidParam string, downloadParam bool) bool { + return !isDir(oidParam) || downloadParam +} + +func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) request { + return request{ RequestCtx: ctx, log: log, } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 1839bf0..3805c2d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -11,9 +11,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -165,7 +165,7 @@ type Handler struct { ownerID *user.ID config Config containerResolver ContainerResolver - tree *tree.Tree + tree layer.TreeService cache *cache.BucketCache workerPool *ants.Pool } @@ -178,7 +178,7 @@ type AppParams struct { Cache *cache.BucketCache } -func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { +func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, @@ -193,77 +193,34 @@ func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Poo // byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - idCnr, _ := c.UserValue("cid").(string) - idObj, _ := url.PathUnescape(c.UserValue("oid").(string)) - - ctx := utils.GetContextFromRequest(c) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With(zap.String("cid", idCnr), zap.String("oid", idObj)) - - bktInfo, err := h.getBucketInfo(ctx, idCnr, log) - if err != nil { - logAndSendBucketError(c, log, err) - return - } - - objID := new(oid.ID) - if err = objID.DecodeString(idObj); err != nil { - log.Error(logs.WrongObjectID, zap.Error(err)) - response.Error(c, "wrong object id", fasthttp.StatusBadRequest) - return - } - - addr := newAddress(bktInfo.CID, *objID) - - f(ctx, *h.newRequest(c, log), addr) +func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) { + addr := newAddress(cnrID, objID) + handler(ctx, req, addr) } // byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that // resolves object address from S3-like path /. -func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - bucketname := c.UserValue("cid").(string) - key := c.UserValue("oid").(string) +func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) { + c, log := req.RequestCtx, req.log - ctx := utils.GetContextFromRequest(c) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With(zap.String("bucketname", bucketname), zap.String("key", key)) - - unescapedKey, err := url.QueryUnescape(key) + foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) if err != nil { logAndSendBucketError(c, log, err) return } - - bktInfo, err := h.getBucketInfo(ctx, bucketname, log) - if err != nil { - logAndSendBucketError(c, log, err) - return - } - - foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey) - if err != nil { - if errors.Is(err, tree.ErrNodeAccessDenied) { - response.Error(c, "Access Denied", fasthttp.StatusForbidden) - } else { - response.Error(c, "object wasn't found", fasthttp.StatusNotFound) - log.Error(logs.GetLatestObjectVersion, zap.Error(err)) - } - return - } - if foundOid.DeleteMarker { + if foundOID.IsDeleteMarker { log.Error(logs.ObjectWasDeleted) response.Error(c, "object deleted", fasthttp.StatusNotFound) return } - addr := newAddress(bktInfo.CID, foundOid.OID) - f(ctx, *h.newRequest(c, log), addr) + addr := newAddress(cnrID, foundOID.OID) + handler(ctx, h.newRequest(c, log), addr) } // byAttribute is a wrapper similar to byNativeAddress. -func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - scid, _ := c.UserValue("cid").(string) +func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Context, request, oid.Address)) { + cidParam, _ := c.UserValue("cid").(string) key, _ := c.UserValue("attr_key").(string) val, _ := c.UserValue("attr_val").(string) @@ -272,21 +229,21 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re key, err := url.QueryUnescape(key) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_key", key), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err)) response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) return } val, err = url.QueryUnescape(val) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_val", val), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err)) response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) return } - log = log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) + log = log.With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val)) - bktInfo, err := h.getBucketInfo(ctx, scid, log) + bktInfo, err := h.getBucketInfo(ctx, cidParam, log) if err != nil { logAndSendBucketError(c, log, err) return @@ -303,11 +260,11 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re return } - var addrObj oid.Address - addrObj.SetContainer(bktInfo.CID) - addrObj.SetObject(objID) + var addr oid.Address + addr.SetContainer(bktInfo.CID) + addr.SetObject(objID) - f(ctx, *h.newRequest(c, log), addrObj) + handler(ctx, h.newRequest(c, log), addr) } func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { @@ -412,7 +369,7 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } -func (h *Handler) browseIndex(c *fasthttp.RequestCtx) { +func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) { if !h.config.IndexPageEnabled() { c.SetStatusCode(fasthttp.StatusNotFound) return @@ -438,18 +395,9 @@ func (h *Handler) browseIndex(c *fasthttp.RequestCtx) { } listFunc := h.getDirObjectsS3 - isNativeList := false - - err = h.tree.CheckSettingsNodeExist(ctx, bktInfo) - if err != nil { - if errors.Is(err, tree.ErrNodeNotFound) { - // tree probe failed, try to use native - listFunc = h.getDirObjectsNative - isNativeList = true - } else { - logAndSendBucketError(c, log, err) - return - } + if isNativeList { + // tree probe failed, trying to use native + listFunc = h.getDirObjectsNative } h.browseObjects(c, browseParams{ diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 86364d9..14f9c98 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -14,8 +14,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" @@ -32,14 +32,29 @@ import ( "go.uber.org/zap" ) -type treeClientMock struct { +type treeServiceMock struct { + system map[string]map[string]*data.BaseNodeVersion } -func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) { - return nil, nil +func newTreeService() *treeServiceMock { + return &treeServiceMock{ + system: make(map[string]map[string]*data.BaseNodeVersion), + } } -func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) { +func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error { + _, ok := t.system["bucket-settings"] + if !ok { + return layer.ErrNodeNotFound + } + return nil +} + +func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeInfo, string, error) { + return nil, "", nil +} + +func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) { return nil, nil } @@ -89,7 +104,7 @@ type handlerContext struct { h *Handler frostfs *TestFrostFS - tree *treeClientMock + tree *treeServiceMock cfg *configMock } @@ -130,14 +145,14 @@ func prepareHandlerContext() (*handlerContext, error) { }), } - treeMock := &treeClientMock{} + treeMock := newTreeService() cfgMock := &configMock{} - workerPool, err := ants.NewPool(1000) + workerPool, err := ants.NewPool(1) if err != nil { return nil, err } - handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool) + handler := New(params, cfgMock, treeMock, workerPool) return &handlerContext{ key: key, diff --git a/internal/handler/head.go b/internal/handler/head.go index ccd6a91..f2e9f38 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -2,11 +2,13 @@ package handler import ( "context" + "errors" "io" "net/http" "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -102,14 +104,36 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) { // HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format. func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { - test, _ := c.UserValue("oid").(string) - var id oid.ID + cidParam, _ := c.UserValue("cid").(string) + oidParam, _ := c.UserValue("oid").(string) - err := id.DecodeString(test) + ctx := utils.GetContextFromRequest(c) + log := utils.GetReqLogOrDefault(ctx, h.log).With( + zap.String("cid", cidParam), + zap.String("oid", oidParam), + ) + + bktInfo, err := h.getBucketInfo(ctx, cidParam, log) if err != nil { - h.byS3Path(c, h.headObject) + logAndSendBucketError(c, log, err) + return + } + checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) + if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + logAndSendBucketError(c, log, checkS3Err) + return + } + + req := h.newRequest(c, log) + + var objID oid.ID + if checkS3Err == nil { + h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject) + } else if err = objID.DecodeString(oidParam); err == nil { + h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { - h.byNativeAddress(c, h.headObject) + logAndSendBucketError(c, log, checkS3Err) + return } } diff --git a/internal/handler/utils.go b/internal/handler/utils.go index b537d64..d09ed23 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -42,16 +42,7 @@ func bearerToken(ctx context.Context) *bearer.Token { } func isDir(name string) bool { - return strings.HasSuffix(name, "/") -} - -func isObjectID(s string) bool { - var objID oid.ID - return objID.DecodeString(s) == nil -} - -func isContainerRoot(key string) bool { - return key == "" + return name == "" || strings.HasSuffix(name, "/") } func loadAttributes(attrs []object.Attribute) map[string]string { diff --git a/internal/api/layer/tree_service.go b/internal/layer/tree_service.go similarity index 64% rename from internal/api/layer/tree_service.go rename to internal/layer/tree_service.go index beb1e7a..ff80543 100644 --- a/internal/api/layer/tree_service.go +++ b/internal/layer/tree_service.go @@ -4,13 +4,15 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) // TreeService provide interface to interact with tree service using s3 data models. type TreeService interface { - GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) + GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) + GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) + CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error } var ( diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 8c8b336..7a04064 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -79,6 +79,11 @@ const ( InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go FailedToUnescapeQuery = "failed to unescape query" + FailedToParseAddressInTreeNode = "failed to parse object addr in tree node" + SettingsNodeInvalidOwnerKey = "settings node: invalid owner key" + SystemNodeHasMultipleIDs = "system node has multiple ids" + FailedToRemoveOldSystemNode = "failed to remove old system node" + BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" diff --git a/internal/service/frostfs/pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go similarity index 100% rename from internal/service/frostfs/pool_wrapper.go rename to internal/service/frostfs/tree_pool_wrapper.go diff --git a/tree/tree.go b/tree/tree.go index 40209a5..bf0aff9 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -6,9 +6,8 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -118,7 +117,7 @@ func (n *treeNode) FileName() (string, bool) { return value, ok } -func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { +func newNodeVersion(node NodeResponse) (*data.NodeVersion, error) { tNode, err := newTreeNode(node) if err != nil { return nil, fmt.Errorf("invalid tree node: %w", err) @@ -127,20 +126,30 @@ func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { return newNodeVersionFromTreeNode(tNode), nil } -func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { +func newNodeVersionFromTreeNode(treeNode *treeNode) *data.NodeVersion { _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) - size, _ := treeNode.Get(sizeKV) - version := &api.NodeVersion{ - BaseNodeVersion: api.BaseNodeVersion{ - OID: treeNode.ObjID, + version := &data.NodeVersion{ + BaseNodeVersion: data.BaseNodeVersion{ + OID: treeNode.ObjID, + IsDeleteMarker: isDeleteMarker, }, - DeleteMarker: isDeleteMarker, - IsPrefixNode: size == "", } return version } +func newNodeInfo(node NodeResponse) data.NodeInfo { + nodeMeta := node.GetMeta() + nodeInfo := data.NodeInfo{ + Meta: make([]data.NodeMeta, 0, len(nodeMeta)), + } + for _, meta := range nodeMeta { + nodeInfo.Meta = append(nodeInfo.Meta, meta) + } + + return nodeInfo +} + func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) { var ( err error @@ -180,7 +189,7 @@ func (m *multiSystemNode) Old() []*treeNode { return m.nodes[1:] } -func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { +func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { nodes, err := c.GetVersions(ctx, cnrID, objectName) if err != nil { return nil, err @@ -210,7 +219,7 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string return c.service.GetNodes(ctx, p) } -func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error { +func (c *Tree) CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error { _, err := c.getSystemNode(ctx, bktInfo, settingsFileName) if err != nil { return err @@ -236,7 +245,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name nodes = filterMultipartNodes(nodes) if len(nodes) == 0 { - return nil, ErrNodeNotFound + return nil, layer.ErrNodeNotFound } return newMultiNode(nodes) @@ -298,14 +307,14 @@ func pathFromName(objectName string) []string { return strings.Split(objectName, separator) } -func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]NodeResponse, string, error) { +func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) { rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) if err != nil { return nil, "", err } subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) if err != nil { - if errors.Is(err, layer.ErrNodeNotFound) { + if errors.Is(err, ErrNodeNotFound) { return nil, "", nil } return nil, "", err @@ -340,14 +349,23 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, nodesMap[fileName] = nodes } - result := make([]NodeResponse, 0, len(subTree)) + result := make([]data.NodeInfo, 0, len(subTree)) for _, nodes := range nodesMap { - result = append(result, nodes...) + result = append(result, nodeResponseToNodeInfo(nodes)...) } return result, strings.TrimSuffix(prefix, tailPrefix), nil } +func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo { + nodesInfo := make([]data.NodeInfo, 0, len(nodes)) + for _, node := range nodes { + nodesInfo = append(nodesInfo, newNodeInfo(node)) + } + + return nodesInfo +} + func (c *Tree) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) ([]uint64, string, error) { rootID := []uint64{0} path := strings.Split(prefix, separator) From a945a947ac7b7833fb97fa81478c550e126f2783 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 16 Dec 2024 15:56:27 +0300 Subject: [PATCH 31/86] [#183] Unlink API.md to README file This is useful for auto-generated document tools which parse docs dir. Signed-off-by: Alex Vanin --- README.md | 139 +---------------------------------------- docs/api.md | 4 +- docs/authentication.md | 108 ++++++++++++++++++++++++++++++++ docs/nns.md | 36 +++++++++++ 4 files changed, 148 insertions(+), 139 deletions(-) create mode 100644 docs/authentication.md create mode 100644 docs/nns.md diff --git a/README.md b/README.md index e1af0eb..adf793c 100644 --- a/README.md +++ b/README.md @@ -217,41 +217,8 @@ Also, in case of downloading, you need to have a file inside a container. ### NNS In all download/upload routes you can use container name instead of its id (`$CID`). +Read more about it in [docs/nns.md](./docs/nns.md). -Steps to start using name resolving: - -1. Enable NNS resolving in config (`rpc_endpoint` must be a valid neo rpc node, see [configs](./config) for other examples): - -```yaml -rpc_endpoint: http://morph-chain.frostfs.devenv:30333 -resolve_order: - - nns -``` - -2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) -you can check if your container (e.g. with `container-name` name) is registered in NNS: - -```shell -$ curl -s --data '{"id":1,"jsonrpc":"2.0","method":"getcontractstate","params":[1]}' \ - http://morph-chain.frostfs.devenv:30333 | jq -r '.result.hash' - -0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 - -$ docker exec -it morph_chain neo-go \ - contract testinvokefunction \ - -r http://morph-chain.frostfs.devenv:30333 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 \ - resolve string:container-name.container int:16 \ - | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ - | base64 -d && echo - -7f3vvkw4iTiS5ZZbu5BQXEmJtETWbi3uUjLNaSs29xrL -``` - -3. Use container name instead of its `$CID`. For example: - -```shell -$ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-name -``` #### Create a container @@ -462,109 +429,7 @@ object ID, like this: #### Authentication -You can always upload files to public containers (open for anyone to put -objects into), but for restricted containers you need to explicitly allow PUT -operations for a request signed with your HTTP Gateway keys. - -If you don't want to manage gateway's secret keys and adjust policies when -gateway configuration changes (new gate, key rotation, etc) or you plan to use -public services, there is an option to let your application backend (or you) to -issue Bearer Tokens and pass them from the client via gate down to FrostFS level -to grant access. - -FrostFS Bearer Token basically is a container owner-signed policy (refer to FrostFS -documentation for more details). There are two options to pass them to gateway: - * "Authorization" header with "Bearer" type and base64-encoded token in - credentials field - * "Bearer" cookie with base64-encoded token contents - -For example, you have a mobile application frontend with a backend part storing -data in FrostFS. When a user authorizes in the mobile app, the backend issues a FrostFS -Bearer token and provides it to the frontend. Then, the mobile app may generate -some data and upload it via any available FrostFS HTTP Gateway by adding -the corresponding header to the upload request. Accessing policy protected data -works the same way. - -##### Example -In order to generate a bearer token, you need to have wallet (which will be used to sign the token) - -1. Suppose you have a container with private policy for wallet key - -``` -$ frostfs-cli container create -r --wallet -policy --basic-acl 0 --await -CID: 9dfzyvq82JnFqp5svxcREf2iy6XNuifYcJPusEDnGK9Z - -$ frostfs-cli ape-manager add -r --wallet \ - --target-type container --target-name 9dfzyvq82JnFqp5svxcREf2iy6XNuifYcJPusEDnGK9Z \ - --rule "allow Object.* RequestCondition:"\$Actor:publicKey"=03b09baabff3f6107c7e9acb8721a6fc5618d45b50247a314d82e548702cce8cd5 *" \ - --chain-id -``` - - -2. Form a Bearer token (10000 is lifetime expiration in epoch) to impersonate - HTTP Gateway request as wallet signed request and save it to **bearer.json**: -``` -{ - "body": { - "allowImpersonate": true, - "lifetime": { - "exp": "10000", - "nbf": "0", - "iat": "0" - } - }, - "signature": null -} -``` - -3. Sign it with the wallet: -``` -$ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w -``` - -4. Encode to base64 to use in header: -``` -$ base64 -w 0 signed.json -# output: Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw== -``` - -After that, the Bearer token can be used: - -``` -$ curl -F 'file=@cat.jpeg;filename=cat.jpeg' -H "Authorization: Bearer Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw==" \ - http://localhost:8082/upload/BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K -# output: -# { -# "object_id": "DhfES9nVrFksxGDD2jQLunGADfrXExxNwqXbDafyBn9X", -# "container_id": "BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K" -# } -``` - -##### Note: Bearer Token owner - -You can specify exact key who can use Bearer Token (gateway wallet address). -To do this, encode wallet address in base64 format - -``` -$ echo 'NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3' | base58 --decode | base64 -# output: NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg== -``` - -Then specify this value in Bearer Token Json -``` -{ - "body": { - "ownerID": { - "value": "NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg==" - }, - ... -``` - -##### Note: Policy override - -Instead of impersonation, you can define the set of policies that will be applied -to the request sender. This allows to restrict access to specific operation and -specific objects without giving full impersonation control to the token user. +Read more about request authentication in [docs/authentication.md](./docs/authemtnication.md) ### Metrics and Pprof diff --git a/docs/api.md b/docs/api.md index f7eb3a4..e59956a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -8,7 +8,7 @@ | `/zip/{cid}/{prefix}` | [Download objects in archive](#download-zip) | **Note:** `cid` parameter can be base58 encoded container ID or container name -(the name must be registered in NNS, see appropriate section in [README](../README.md#nns)). +(the name must be registered in NNS, see appropriate section in [nns.md](./nns.md)). Route parameters can be: @@ -18,7 +18,7 @@ Route parameters can be: ### Bearer token -All routes can accept [bearer token](../README.md#authentication) from: +All routes can accept [bearer token](./authentication.md) from: * `Authorization` header with `Bearer` type and base64-encoded token in credentials field diff --git a/docs/authentication.md b/docs/authentication.md new file mode 100644 index 0000000..d8bb235 --- /dev/null +++ b/docs/authentication.md @@ -0,0 +1,108 @@ +# Request authentication + +HTTP Gateway does not authorize requests. Gateway converts HTTP request to a +FrostFS request and signs it with its own private key. + +You can always upload files to public containers (open for anyone to put +objects into), but for restricted containers you need to explicitly allow PUT +operations for a request signed with your HTTP Gateway keys. + +If you don't want to manage gateway's secret keys and adjust policies when +gateway configuration changes (new gate, key rotation, etc) or you plan to use +public services, there is an option to let your application backend (or you) to +issue Bearer Tokens and pass them from the client via gate down to FrostFS level +to grant access. + +FrostFS Bearer Token basically is a container owner-signed policy (refer to FrostFS +documentation for more details). There are two options to pass them to gateway: +* "Authorization" header with "Bearer" type and base64-encoded token in + credentials field +* "Bearer" cookie with base64-encoded token contents + +For example, you have a mobile application frontend with a backend part storing +data in FrostFS. When a user authorizes in the mobile app, the backend issues a FrostFS +Bearer token and provides it to the frontend. Then, the mobile app may generate +some data and upload it via any available FrostFS HTTP Gateway by adding +the corresponding header to the upload request. Accessing policy protected data +works the same way. + +##### Example +In order to generate a bearer token, you need to have wallet (which will be used to sign the token) + +1. Suppose you have a container with private policy for wallet key + +``` +$ frostfs-cli container create -r --wallet -policy --basic-acl 0 --await +CID: 9dfzyvq82JnFqp5svxcREf2iy6XNuifYcJPusEDnGK9Z + +$ frostfs-cli ape-manager add -r --wallet \ + --target-type container --target-name 9dfzyvq82JnFqp5svxcREf2iy6XNuifYcJPusEDnGK9Z \ + --rule "allow Object.* RequestCondition:"\$Actor:publicKey"=03b09baabff3f6107c7e9acb8721a6fc5618d45b50247a314d82e548702cce8cd5 *" \ + --chain-id +``` + + +2. Form a Bearer token (10000 is lifetime expiration in epoch) to impersonate + HTTP Gateway request as wallet signed request and save it to **bearer.json**: +``` +{ + "body": { + "allowImpersonate": true, + "lifetime": { + "exp": "10000", + "nbf": "0", + "iat": "0" + } + }, + "signature": null +} +``` + +3. Sign it with the wallet: +``` +$ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w +``` + +4. Encode to base64 to use in header: +``` +$ base64 -w 0 signed.json +# output: Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw== +``` + +After that, the Bearer token can be used: + +``` +$ curl -F 'file=@cat.jpeg;filename=cat.jpeg' -H "Authorization: Bearer Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw==" \ + http://localhost:8082/upload/BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K +# output: +# { +# "object_id": "DhfES9nVrFksxGDD2jQLunGADfrXExxNwqXbDafyBn9X", +# "container_id": "BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K" +# } +``` + +##### Note: Bearer Token owner + +You can specify exact key who can use Bearer Token (gateway wallet address). +To do this, encode wallet address in base64 format + +``` +$ echo 'NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3' | base58 --decode | base64 +# output: NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg== +``` + +Then specify this value in Bearer Token Json +``` +{ + "body": { + "ownerID": { + "value": "NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg==" + }, + ... +``` + +##### Note: Policy override + +Instead of impersonation, you can define the set of policies that will be applied +to the request sender. This allows to restrict access to specific operation and +specific objects without giving full impersonation control to the token user. diff --git a/docs/nns.md b/docs/nns.md new file mode 100644 index 0000000..acb9f21 --- /dev/null +++ b/docs/nns.md @@ -0,0 +1,36 @@ +# Nicename Resolving with NNS + +Steps to start using name resolving: + +1. Enable NNS resolving in config (`rpc_endpoint` must be a valid neo rpc node, see [configs](./config) for other examples): + +```yaml +rpc_endpoint: http://morph-chain.frostfs.devenv:30333 +resolve_order: + - nns +``` + +2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) + you can check if your container (e.g. with `container-name` name) is registered in NNS: + +```shell +$ curl -s --data '{"id":1,"jsonrpc":"2.0","method":"getcontractstate","params":[1]}' \ + http://morph-chain.frostfs.devenv:30333 | jq -r '.result.hash' + +0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 + +$ docker exec -it morph_chain neo-go \ + contract testinvokefunction \ + -r http://morph-chain.frostfs.devenv:30333 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 \ + resolve string:container-name.container int:16 \ + | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ + | base64 -d && echo + +7f3vvkw4iTiS5ZZbu5BQXEmJtETWbi3uUjLNaSs29xrL +``` + +3. Use container name instead of its `$CID`. For example: + +```shell +$ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-name +``` From a658f3adc0ec2bf7f1dbb13701360eeb5c4a4470 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 16 Dec 2024 13:01:50 +0300 Subject: [PATCH 32/86] [#181] index_page: Ignore deleted objects in versioned buckets Signed-off-by: Nikita Zinkevich --- internal/handler/browse.go | 42 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/internal/handler/browse.go b/internal/handler/browse.go index 2938b5c..64ad1f5 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -22,12 +22,13 @@ import ( ) const ( - dateFormat = "02-01-2006 15:04" - attrOID = "OID" - attrCreated = "Created" - attrFileName = "FileName" - attrFilePath = "FilePath" - attrSize = "Size" + dateFormat = "02-01-2006 15:04" + attrOID = "OID" + attrCreated = "Created" + attrFileName = "FileName" + attrFilePath = "FilePath" + attrSize = "Size" + attrDeleteMarker = "IsDeleteMarker" ) type ( @@ -39,23 +40,25 @@ type ( Objects []ResponseObject } ResponseObject struct { - OID string - Created string - FileName string - FilePath string - Size string - IsDir bool - GetURL string + OID string + Created string + FileName string + FilePath string + Size string + IsDir bool + GetURL string + IsDeleteMarker bool } ) func newListObjectsResponseS3(attrs map[string]string) ResponseObject { return ResponseObject{ - Created: formatTimestamp(attrs[attrCreated]), - OID: attrs[attrOID], - FileName: attrs[attrFileName], - Size: attrs[attrSize], - IsDir: attrs[attrOID] == "", + Created: formatTimestamp(attrs[attrCreated]), + OID: attrs[attrOID], + FileName: attrs[attrFileName], + Size: attrs[attrSize], + IsDir: attrs[attrOID] == "", + IsDeleteMarker: attrs[attrDeleteMarker] == "true", } } @@ -179,6 +182,9 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn attrs[m.GetKey()] = string(m.GetValue()) } obj := newListObjectsResponseS3(attrs) + if obj.IsDeleteMarker { + continue + } obj.FilePath = prefix + obj.FileName obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath) result.objects = append(result.objects, obj) From d32ac4b5370f93ea488cfa2191f1ed15e35800ba Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 13:49:13 +0300 Subject: [PATCH 33/86] Release v0.32.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 16 +++++++++++++++- VERSION | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc422d6..e528b8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ This document outlines major changes between releases. ## [Unreleased] +## [0.32.0] - Khumbu - 2024-12-20 + +### Fixed +- Getting S3 object with FrostFS Object ID-like key (#166) +- Ignore delete marked objects in versioned bucket in index page (#181) + +### Added +- Metric of dropped logs by log sampler (#150) +- Fallback FileName attribute search during FilePath attribute search (#174) + +### Changed +- Updated tree service pool without api-go dependency (#178) + ## [0.31.0] - Rongbuk - 2024-11-20 ### Fixed @@ -170,4 +183,5 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.30.2]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.1...v0.30.2 [0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.2...v0.30.3 [0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.3...v0.31.0 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...master \ No newline at end of file +[0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...v0.32.0 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.0...master \ No newline at end of file diff --git a/VERSION b/VERSION index 7021025..420000f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.31.0 +v0.32.0 From a4e3767d4bb2c6853bd52e1eabffb1729d5dd1fd Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Wed, 18 Dec 2024 14:31:12 +0300 Subject: [PATCH 34/86] [#175] Adopt 1.6.* aio versoins in integration tests Signed-off-by: Roman Loginov --- cmd/http-gw/integration_test.go | 45 +++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index dd4d60f..6a4d428 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -14,6 +14,7 @@ import ( "net/http" "os" "sort" + "strings" "testing" "time" @@ -54,6 +55,7 @@ func TestIntegration(t *testing.T) { "1.2.7", "1.3.0", "1.5.0", + "1.6.5", } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) @@ -70,21 +72,26 @@ func TestIntegration(t *testing.T) { ctx, cancel2 := context.WithCancel(rootCtx) aioContainer := createDockerContainer(ctx, t, aioImage+version) + if strings.HasPrefix(version, "1.6") { + registerUser(t, ctx, aioContainer, file.Name()) + } + + // See the logs from the command execution. server, cancel := runServer(file.Name()) clientPool := getPool(ctx, t, key) - CID, err := createContainer(ctx, t, clientPool, ownerID, version) + CID, err := createContainer(ctx, t, clientPool, ownerID) require.NoError(t, err, version) token := makeBearerToken(t, key, ownerID, version) - t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) }) + t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) }) t.Run("put with bearer token in header"+version, func(t *testing.T) { putWithBearerTokenInHeader(ctx, t, clientPool, CID, token) }) t.Run("put with bearer token in cookie"+version, func(t *testing.T) { putWithBearerTokenInCookie(ctx, t, clientPool, CID, token) }) t.Run("put with duplicate keys "+version, func(t *testing.T) { putWithDuplicateKeys(t, CID) }) - t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID, version) }) - t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID, version) }) - t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID, version) }) - t.Run("test namespaces "+version, func(t *testing.T) { checkNamespaces(ctx, t, clientPool, ownerID, CID, version) }) + t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID) }) + t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID) }) + t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID) }) + t.Run("test namespaces "+version, func(t *testing.T) { checkNamespaces(ctx, t, clientPool, ownerID, CID) }) cancel() server.Wait() @@ -107,7 +114,7 @@ func runServer(pathToWallet string) (App, context.CancelFunc) { return application, cancel } -func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID, version string) { +func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID) { url := testHost + "/upload/" + CID.String() makePutRequestAndCheck(ctx, t, p, CID, url) @@ -255,7 +262,7 @@ func putWithDuplicateKeys(t *testing.T, CID cid.ID) { require.Equal(t, http.StatusBadRequest, resp.StatusCode) } -func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { +func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID) { content := "content of file" attributes := map[string]string{ "some-attr": "some-get-value", @@ -302,7 +309,7 @@ func checkGetByAttrResponse(t *testing.T, resp *http.Response, content string, a } } -func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { +func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID) { keyAttr, valAttr := "some-attr", "some-get-by-attr-value" content := "content of file" attributes := map[string]string{keyAttr: valAttr} @@ -324,7 +331,7 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID checkGetByAttrResponse(t, resp, content, expectedAttr) } -func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { +func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID) { names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} contents := []string{"content of file1", "content of file2"} attributes1 := map[string]string{object.AttributeFilePath: names[0]} @@ -389,7 +396,7 @@ func checkZip(t *testing.T, data []byte, length int64, names, contents []string) } } -func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { +func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID) { content := "content of file" attributes := map[string]string{ "some-attr": "some-get-value", @@ -426,7 +433,7 @@ func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, o func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container { req := testcontainers.ContainerRequest{ Image: image, - WaitingFor: wait.NewLogStrategy("aio container started").WithStartupTimeout(30 * time.Second), + WaitingFor: wait.NewLogStrategy("aio container started").WithStartupTimeout(2 * time.Minute), Name: "aio", Hostname: "aio", NetworkMode: "host", @@ -466,7 +473,7 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool return clientPool } -func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, version string) (cid.ID, error) { +func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID) (cid.ID, error) { var policy netmap.PlacementPolicy err := policy.DecodeString("REP 1") require.NoError(t, err) @@ -526,6 +533,18 @@ func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID return id.ObjectID } +func registerUser(t *testing.T, ctx context.Context, aioContainer testcontainers.Container, pathToWallet string) { + err := aioContainer.CopyFileToContainer(ctx, pathToWallet, "/usr/wallet.json", 644) + require.NoError(t, err) + + _, err = aioContainer.Exec(ctx, []string{ + "/usr/bin/frostfs-s3-authmate", "register-user", + "--wallet", "/usr/wallet.json", + "--rpc-endpoint", "http://localhost:30333", + "--contract-wallet", "/config/s3-gw-wallet.json"}) + require.NoError(t, err) +} + func makeBearerToken(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) string { tkn := new(bearer.Token) tkn.ForUser(ownerID) From 9551f34f00ffa1367ebe703609647b42ba3b1aa8 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Fri, 29 Nov 2024 09:48:44 +0300 Subject: [PATCH 35/86] [#163] Support JSON bearer token Signed-off-by: Roman Loginov --- cmd/http-gw/integration_test.go | 22 ++++++++++----- tokens/bearer-token.go | 18 +++++++++---- tokens/bearer-token_test.go | 48 ++++++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 6a4d428..4c20546 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -82,11 +82,13 @@ func TestIntegration(t *testing.T) { CID, err := createContainer(ctx, t, clientPool, ownerID) require.NoError(t, err, version) - token := makeBearerToken(t, key, ownerID, version) + jsonToken, binaryToken := makeBearerTokens(t, key, ownerID, version) t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) }) - t.Run("put with bearer token in header"+version, func(t *testing.T) { putWithBearerTokenInHeader(ctx, t, clientPool, CID, token) }) - t.Run("put with bearer token in cookie"+version, func(t *testing.T) { putWithBearerTokenInCookie(ctx, t, clientPool, CID, token) }) + t.Run("put with json bearer token in header"+version, func(t *testing.T) { putWithBearerTokenInHeader(ctx, t, clientPool, CID, jsonToken) }) + t.Run("put with json bearer token in cookie"+version, func(t *testing.T) { putWithBearerTokenInCookie(ctx, t, clientPool, CID, jsonToken) }) + t.Run("put with binary bearer token in header"+version, func(t *testing.T) { putWithBearerTokenInHeader(ctx, t, clientPool, CID, binaryToken) }) + t.Run("put with binary bearer token in cookie"+version, func(t *testing.T) { putWithBearerTokenInCookie(ctx, t, clientPool, CID, binaryToken) }) t.Run("put with duplicate keys "+version, func(t *testing.T) { putWithDuplicateKeys(t, CID) }) t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID) }) t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID) }) @@ -545,7 +547,7 @@ func registerUser(t *testing.T, ctx context.Context, aioContainer testcontainers require.NoError(t, err) } -func makeBearerToken(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) string { +func makeBearerTokens(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) (jsonTokenBase64, binaryTokenBase64 string) { tkn := new(bearer.Token) tkn.ForUser(ownerID) tkn.SetExp(10000) @@ -559,10 +561,16 @@ func makeBearerToken(t *testing.T, key *keys.PrivateKey, ownerID user.ID, versio err := tkn.Sign(key.PrivateKey) require.NoError(t, err) - t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) - require.NotEmpty(t, t64) + jsonToken, err := tkn.MarshalJSON() + require.NoError(t, err) - return t64 + jsonTokenBase64 = base64.StdEncoding.EncodeToString(jsonToken) + binaryTokenBase64 = base64.StdEncoding.EncodeToString(tkn.Marshal()) + + require.NotEmpty(t, jsonTokenBase64) + require.NotEmpty(t, binaryTokenBase64) + + return } func makeTempWallet(t *testing.T, key *keys.PrivateKey, path string) { diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index 880a100..24ffcbe 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -82,14 +82,22 @@ func fetchBearerToken(ctx *fasthttp.RequestCtx) (*bearer.Token, error) { tkn = new(bearer.Token) ) for _, parse := range []fromHandler{BearerTokenFromHeader, BearerTokenFromCookie} { - if buf = parse(&ctx.Request.Header); buf == nil { + buf = parse(&ctx.Request.Header) + if buf == nil { continue - } else if data, err := base64.StdEncoding.DecodeString(string(buf)); err != nil { + } + + data, err := base64.StdEncoding.DecodeString(string(buf)) + if err != nil { lastErr = fmt.Errorf("can't base64-decode bearer token: %w", err) continue - } else if err = tkn.Unmarshal(data); err != nil { - lastErr = fmt.Errorf("can't unmarshal bearer token: %w", err) - continue + } + + if err = tkn.Unmarshal(data); err != nil { + if err = tkn.UnmarshalJSON(data); err != nil { + lastErr = fmt.Errorf("can't unmarshal bearer token: %w", err) + continue + } } return tkn, nil diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index 6fb3bf4..60e9ea2 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -98,8 +98,14 @@ func TestFetchBearerToken(t *testing.T) { tkn := new(bearer.Token) tkn.ForUser(uid) - t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) - require.NotEmpty(t, t64) + jsonToken, err := tkn.MarshalJSON() + require.NoError(t, err) + + jsonTokenBase64 := base64.StdEncoding.EncodeToString(jsonToken) + binaryTokenBase64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) + + require.NotEmpty(t, jsonTokenBase64) + require.NotEmpty(t, binaryTokenBase64) cases := []struct { name string @@ -143,25 +149,47 @@ func TestFetchBearerToken(t *testing.T) { error: "can't unmarshal bearer token", }, { - name: "bad header, but good cookie", + name: "bad header, but good cookie with binary token", header: "dGVzdAo=", - cookie: t64, + cookie: binaryTokenBase64, expect: tkn, }, { - name: "bad cookie, but good header", - header: t64, + name: "bad cookie, but good header with binary token", + header: binaryTokenBase64, cookie: "dGVzdAo=", expect: tkn, }, { - name: "ok for header", - header: t64, + name: "bad header, but good cookie with json token", + header: "dGVzdAo=", + cookie: jsonTokenBase64, expect: tkn, }, { - name: "ok for cookie", - cookie: t64, + name: "bad cookie, but good header with json token", + header: jsonTokenBase64, + cookie: "dGVzdAo=", + expect: tkn, + }, + { + name: "ok for header with binary token", + header: binaryTokenBase64, + expect: tkn, + }, + { + name: "ok for cookie with binary token", + cookie: binaryTokenBase64, + expect: tkn, + }, + { + name: "ok for header with json token", + header: jsonTokenBase64, + expect: tkn, + }, + { + name: "ok for cookie with json token", + cookie: jsonTokenBase64, expect: tkn, }, } From e1b670a727529cb2232af4cedd1f6dc26d357559 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Thu, 9 Jan 2025 10:48:07 +0300 Subject: [PATCH 36/86] [#192] Build and host OCI images on our own infra Similar to https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/issues/587 this PR introduces a CI pipeline that builds Docker images and pushes them to our selfhosted registry. Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/oci-image.yml | 27 +++++++++++++++++++++++++++ Makefile | 2 +- README.md | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 .forgejo/workflows/oci-image.yml diff --git a/.forgejo/workflows/oci-image.yml b/.forgejo/workflows/oci-image.yml new file mode 100644 index 0000000..c5c0a2e --- /dev/null +++ b/.forgejo/workflows/oci-image.yml @@ -0,0 +1,27 @@ +on: + pull_request: + push: + workflow_dispatch: + +jobs: + image: + name: OCI image + runs-on: docker + container: git.frostfs.info/truecloudlab/env:oci-image-builder-bookworm + steps: + - name: Clone git repo + uses: actions/checkout@v3 + + - name: Build OCI image + run: make image + + - name: Push image to OCI registry + run: | + echo "$REGISTRY_PASSWORD" \ + | docker login --username truecloudlab --password-stdin git.frostfs.info + make image-push + if: >- + startsWith(github.ref, 'refs/tags/v') && + (github.event_name == 'workflow_dispatch' || github.event_name == 'push') + env: + REGISTRY_PASSWORD: ${{secrets.FORGEJO_OCI_REGISTRY_PUSH_TOKEN}} diff --git a/Makefile b/Makefile index c1f4f50..5b9e5bf 100755 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ LINT_VERSION ?= 1.60.3 TRUECLOUDLAB_LINT_VERSION ?= 0.0.6 BUILD ?= $(shell date -u --iso=seconds) -HUB_IMAGE ?= truecloudlab/frostfs-http-gw +HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs-http-gw HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" METRICS_DUMP_OUT ?= ./metrics-dump.json diff --git a/README.md b/README.md index adf793c..9c17c2a 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ version Show current version ``` Or you can also use a [Docker -image](https://hub.docker.com/r/truecloudlab/frostfs-http-gw) provided for the released +image](https://git.frostfs.info/TrueCloudLab/-/packages/container/frostfs-http-gw) provided for the released (and occasionally unreleased) versions of the gateway (`:latest` points to the latest stable release). From 1db62f9d953e79ca269951215280adb45ee12a53 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Fri, 20 Dec 2024 09:45:42 +0300 Subject: [PATCH 37/86] [#185] Update SDK to support new tree/pool version Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 6 ++- cmd/http-gw/settings.go | 58 +++++++++++++++--------- config/config.env | 4 ++ config/config.yaml | 5 +++ docs/gate-configuration.md | 14 +++--- go.mod | 13 +++++- go.sum | 36 ++++++++++++++- internal/cache/buckets.go | 51 ++++++++++++++++++--- internal/cache/netmap.go | 65 +++++++++++++++++++++++++++ internal/data/info.go | 2 + internal/handler/handler.go | 1 + internal/handler/handler_test.go | 2 +- internal/logs/logs.go | 1 + internal/service/frostfs/frostfs.go | 10 +++++ internal/service/frostfs/source.go | 69 +++++++++++++++++++++++++++++ 15 files changed, 299 insertions(+), 38 deletions(-) create mode 100644 internal/cache/netmap.go create mode 100644 internal/service/frostfs/source.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 23a752a..3386536 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -65,6 +65,7 @@ type ( services []*metrics.Service settings *appSettings loggerSettings *loggerSettings + bucketCache *cache.BucketCache servers []Server unbindServers []ServerInfo @@ -134,6 +135,7 @@ func newApp(ctx context.Context, v *viper.Viper) App { loggerSettings: logSettings, webServer: new(fasthttp.Server), webDone: make(chan struct{}), + bucketCache: cache.NewBucketCache(getBucketCacheOptions(v, log.logger), v.GetBool(cfgFeaturesTreePoolNetmapSupport)), } a.initAppSettings() @@ -151,7 +153,7 @@ func newApp(ctx context.Context, v *viper.Viper) App { a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - a.pool, a.treePool, a.key = getPools(ctx, a.log, a.cfg, a.settings.dialerSource) + a.initPools(ctx) var owner user.ID user.IDFromKey(&owner, a.key.PrivateKey.PublicKey) @@ -839,7 +841,7 @@ func (a *app) AppParams() *handler.AppParams { FrostFS: frostfs.NewFrostFS(a.pool), Owner: a.owner, Resolver: a.resolver, - Cache: cache.NewBucketCache(getCacheOptions(a.cfg, a.log)), + Cache: a.bucketCache, } } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 62ef83e..691e9ba 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -17,12 +17,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/zapjournald" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/ssgreg/journald" @@ -144,6 +144,7 @@ const ( // Caching. cfgBucketsCacheLifetime = "cache.buckets.lifetime" cfgBucketsCacheSize = "cache.buckets.size" + cfgNetmapCacheLifetime = "cache.netmap.lifetime" // Bucket resolving options. cfgResolveNamespaceHeader = "resolve_bucket.namespace_header" @@ -166,6 +167,7 @@ const ( // Feature. cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" + cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" // Command line args. cmdHelp = "help" @@ -611,10 +613,10 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { return servers } -func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper, dialSource *internalnet.DialerSource) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { - key, err := getFrostFSKey(cfg, logger) +func (a *app) initPools(ctx context.Context) { + key, err := getFrostFSKey(a.cfg, a.log) if err != nil { - logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) + a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) } var prm pool.InitParameters @@ -622,77 +624,83 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper, dialSou prm.SetKey(&key.PrivateKey) prmTree.SetKey(key) - logger.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) + a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) - for _, peer := range fetchPeers(logger, cfg) { + for _, peer := range fetchPeers(a.log, a.cfg) { prm.AddNode(peer) prmTree.AddNode(peer) } - connTimeout := cfg.GetDuration(cfgConTimeout) + connTimeout := a.cfg.GetDuration(cfgConTimeout) if connTimeout <= 0 { connTimeout = defaultConnectTimeout } prm.SetNodeDialTimeout(connTimeout) prmTree.SetNodeDialTimeout(connTimeout) - streamTimeout := cfg.GetDuration(cfgStreamTimeout) + streamTimeout := a.cfg.GetDuration(cfgStreamTimeout) if streamTimeout <= 0 { streamTimeout = defaultStreamTimeout } prm.SetNodeStreamTimeout(streamTimeout) prmTree.SetNodeStreamTimeout(streamTimeout) - healthCheckTimeout := cfg.GetDuration(cfgReqTimeout) + healthCheckTimeout := a.cfg.GetDuration(cfgReqTimeout) if healthCheckTimeout <= 0 { healthCheckTimeout = defaultRequestTimeout } prm.SetHealthcheckTimeout(healthCheckTimeout) prmTree.SetHealthcheckTimeout(healthCheckTimeout) - rebalanceInterval := cfg.GetDuration(cfgRebalance) + rebalanceInterval := a.cfg.GetDuration(cfgRebalance) if rebalanceInterval <= 0 { rebalanceInterval = defaultRebalanceTimer } prm.SetClientRebalanceInterval(rebalanceInterval) prmTree.SetClientRebalanceInterval(rebalanceInterval) - errorThreshold := cfg.GetUint32(cfgPoolErrorThreshold) + errorThreshold := a.cfg.GetUint32(cfgPoolErrorThreshold) if errorThreshold <= 0 { errorThreshold = defaultPoolErrorThreshold } prm.SetErrorThreshold(errorThreshold) - prm.SetLogger(logger) - prmTree.SetLogger(logger) + prm.SetLogger(a.log) + prmTree.SetLogger(a.log) - prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgTreePoolMaxAttempts)) + prmTree.SetMaxRequestAttempts(a.cfg.GetInt(cfgTreePoolMaxAttempts)) interceptors := []grpc.DialOption{ grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), - grpc.WithContextDialer(dialSource.GrpcContextDialer()), + grpc.WithContextDialer(a.settings.dialerSource.GrpcContextDialer()), } prm.SetGRPCDialOptions(interceptors...) prmTree.SetGRPCDialOptions(interceptors...) p, err := pool.NewPool(prm) if err != nil { - logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) } if err = p.Dial(ctx); err != nil { - logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + } + + if a.cfg.GetBool(cfgFeaturesTreePoolNetmapSupport) { + prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p), cache.NewNetmapCache(getNetmapCacheOptions(a.cfg, a.log)), a.bucketCache, a.log)) } treePool, err := treepool.NewPool(prmTree) if err != nil { - logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) } if err = treePool.Dial(ctx); err != nil { - logger.Fatal(logs.FailedToDialTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err)) } - return p, treePool, key + a.pool = p + a.treePool = treePool + a.key = key } func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam { @@ -733,7 +741,7 @@ func fetchSoftMemoryLimit(cfg *viper.Viper) int64 { return int64(softMemoryLimit) } -func getCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { +func getBucketCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { cacheCfg := cache.DefaultBucketConfig(l) cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgBucketsCacheLifetime, cacheCfg.Lifetime) @@ -742,6 +750,14 @@ func getCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { return cacheCfg } +func getNetmapCacheOptions(v *viper.Viper, l *zap.Logger) *cache.NetmapCacheConfig { + cacheCfg := cache.DefaultNetmapConfig(l) + + cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgNetmapCacheLifetime, cacheCfg.Lifetime) + + return cacheCfg +} + func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration { if v.IsSet(cfgEntry) { lifetime := v.GetDuration(cfgEntry) diff --git a/config/config.env b/config/config.env index 2822357..171889f 100644 --- a/config/config.env +++ b/config/config.env @@ -121,6 +121,8 @@ HTTP_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576 # Cache which contains mapping of bucket name to bucket info HTTP_GW_CACHE_BUCKETS_LIFETIME=1m HTTP_GW_CACHE_BUCKETS_SIZE=1000 +# Cache which stores netmap +HTTP_GW_CACHE_NETMAP_LIFETIME=1m # Header to determine zone to resolve bucket name HTTP_GW_RESOLVE_BUCKET_NAMESPACE_HEADER=X-Frostfs-Namespace @@ -162,3 +164,5 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl # Enable using fallback path to search for a object by attribute HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false +# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service +HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true diff --git a/config/config.yaml b/config/config.yaml index 6296bd9..eee84e5 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -143,6 +143,9 @@ cache: buckets: lifetime: 1m size: 1000 + # Cache which stores netmap + netmap: + lifetime: 1m resolve_bucket: namespace_header: X-Frostfs-Namespace @@ -176,3 +179,5 @@ multinet: features: # Enable using fallback path to search for a object by attribute enable_filepath_fallback: false + # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service + tree_pool_netmap_support: true diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 7476f5d..ce7c0c7 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -339,12 +339,14 @@ cache: buckets: lifetime: 1m size: 1000 - + netmap: + lifetime: 1m ``` -| Parameter | Type | Default value | Description | -|-----------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------------------------| -| `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`
`size: 1000` | Cache which contains mapping of bucket name to bucket info. | +| Parameter | Type | Default value | Description | +|-----------|-----------------------------------|---------------------------------|---------------------------------------------------------------------------| +| `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`
`size: 1000` | Cache which contains mapping of bucket name to bucket info. | +| `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. | #### `cache` subsection @@ -465,8 +467,10 @@ Contains parameters for enabling features. ```yaml features: enable_filepath_fallback: true + tree_pool_netmap_support: true ``` | Parameter | Type | SIGHUP reload | Default value | Description | -| ----------------------------------- | ------ | ------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. | +| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | diff --git a/go.mod b/go.mod index 3dd27b8..0b74841 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 @@ -65,16 +65,25 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/ipfs/go-cid v0.0.7 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/sys/mount v0.3.2 // indirect github.com/moby/sys/mountinfo v0.6.1 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr v0.14.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect @@ -89,6 +98,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect github.com/sirupsen/logrus v1.8.1 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -116,4 +126,5 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 7f43c86..06bdd7d 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d h1:FpXI+mOrmJk3t2MKQFZuhLjCHDyDeo5rtP1WXl7gUWc= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae h1:7gvuOTmS3oaOM79JkHWWlsvGqIRqsum5KnOI1TYqfn0= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae/go.mod h1:dbWUc5jOBTXVvssCLCYxkkSTL9jgLr1KruGP2FMAfiM= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -519,6 +519,8 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -544,6 +546,8 @@ github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -582,6 +586,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -610,9 +618,28 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU= +github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -762,6 +789,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= @@ -1121,6 +1150,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1407,6 +1437,8 @@ k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/cache/buckets.go b/internal/cache/buckets.go index f8e6d88..2fa8f25 100644 --- a/internal/cache/buckets.go +++ b/internal/cache/buckets.go @@ -6,14 +6,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/bluele/gcache" "go.uber.org/zap" ) // BucketCache contains cache with objects and the lifetime of cache entries. type BucketCache struct { - cache gcache.Cache - logger *zap.Logger + cache gcache.Cache + cidCache gcache.Cache + logger *zap.Logger } // Config stores expiration params for cache. @@ -40,14 +42,45 @@ func DefaultBucketConfig(logger *zap.Logger) *Config { } // NewBucketCache creates an object of BucketCache. -func NewBucketCache(config *Config) *BucketCache { - gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() - return &BucketCache{cache: gc, logger: config.Logger} +func NewBucketCache(config *Config, cidCache bool) *BucketCache { + cache := &BucketCache{ + cache: gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build(), + logger: config.Logger, + } + + if cidCache { + cache.cidCache = gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + } + return cache } // Get returns a cached object. func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo { - entry, err := o.cache.Get(formKey(ns, bktName)) + return o.get(formKey(ns, bktName)) +} + +func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo { + if o.cidCache == nil { + return nil + } + + entry, err := o.cidCache.Get(cnrID) + if err != nil { + return nil + } + + key, ok := entry.(string) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", key))) + return nil + } + + return o.get(key) +} + +func (o *BucketCache) get(key string) *data.BucketInfo { + entry, err := o.cache.Get(key) if err != nil { return nil } @@ -64,6 +97,12 @@ func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo { // Put puts an object to cache. func (o *BucketCache) Put(bkt *data.BucketInfo) error { + if o.cidCache != nil { + if err := o.cidCache.Set(bkt.CID, formKey(bkt.Zone, bkt.Name)); err != nil { + return err + } + } + return o.cache.Set(formKey(bkt.Zone, bkt.Name), bkt) } diff --git a/internal/cache/netmap.go b/internal/cache/netmap.go new file mode 100644 index 0000000..6d91fe7 --- /dev/null +++ b/internal/cache/netmap.go @@ -0,0 +1,65 @@ +package cache + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/bluele/gcache" + "go.uber.org/zap" +) + +type ( + // NetmapCache provides cache for netmap. + NetmapCache struct { + cache gcache.Cache + logger *zap.Logger + } + + // NetmapCacheConfig stores expiration params for cache. + NetmapCacheConfig struct { + Lifetime time.Duration + Logger *zap.Logger + } +) + +const ( + DefaultNetmapCacheLifetime = time.Minute + netmapCacheSize = 1 + netmapKey = "netmap" +) + +// DefaultNetmapConfig returns new default cache expiration values. +func DefaultNetmapConfig(logger *zap.Logger) *NetmapCacheConfig { + return &NetmapCacheConfig{ + Lifetime: DefaultNetmapCacheLifetime, + Logger: logger, + } +} + +// NewNetmapCache creates an object of NetmapCache. +func NewNetmapCache(config *NetmapCacheConfig) *NetmapCache { + gc := gcache.New(netmapCacheSize).LRU().Expiration(config.Lifetime).Build() + return &NetmapCache{cache: gc, logger: config.Logger} +} + +func (c *NetmapCache) Get() *netmap.NetMap { + entry, err := c.cache.Get(netmapKey) + if err != nil { + return nil + } + + result, ok := entry.(netmap.NetMap) + if !ok { + c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result))) + return nil + } + + return &result +} + +func (c *NetmapCache) Put(nm netmap.NetMap) error { + return c.cache.Set(netmapKey, nm) +} diff --git a/internal/data/info.go b/internal/data/info.go index d99ca49..f5c80d6 100644 --- a/internal/data/info.go +++ b/internal/data/info.go @@ -2,6 +2,7 @@ package data import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) type BucketInfo struct { @@ -9,4 +10,5 @@ type BucketInfo struct { Zone string // container zone from system attribute CID cid.ID HomomorphicHashDisabled bool + PlacementPolicy netmap.PlacementPolicy } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 3805c2d..1150f45 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -365,6 +365,7 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket } bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(*res) + bktInfo.PlacementPolicy = res.PlacementPolicy() return bktInfo, err } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 14f9c98..e1bc010 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -142,7 +142,7 @@ func prepareHandlerContext() (*handlerContext, error) { Size: 1, Lifetime: 1, Logger: logger, - }), + }, false), } treeMock := newTreeService() diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 7a04064..f9b13b1 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -93,4 +93,5 @@ const ( FailedToLoadMultinetConfig = "failed to load multinet config" MultinetConfigWontBeUpdated = "multinet config won't be updated" ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" + CouldntCacheNetmap = "couldn't cache netmap" ) diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index c7e56a4..b218976 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" @@ -173,6 +174,15 @@ func (x *FrostFS) GetEpochDurations(ctx context.Context) (*utils.EpochDurations, return res, nil } +func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) { + netmapSnapshot, err := x.pool.NetMapSnapshot(ctx) + if err != nil { + return netmapSnapshot, handleObjectError("get netmap via connection pool", err) + } + + return netmapSnapshot, nil +} + // ResolverFrostFS represents virtual connection to the FrostFS network. // It implements resolver.FrostFS. type ResolverFrostFS struct { diff --git a/internal/service/frostfs/source.go b/internal/service/frostfs/source.go new file mode 100644 index 0000000..de6c681 --- /dev/null +++ b/internal/service/frostfs/source.go @@ -0,0 +1,69 @@ +package frostfs + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "go.uber.org/zap" +) + +type Source struct { + frostFS *FrostFS + netmapCache *cache.NetmapCache + bucketCache *cache.BucketCache + log *zap.Logger +} + +func NewSource(frostFS *FrostFS, netmapCache *cache.NetmapCache, bucketCache *cache.BucketCache, log *zap.Logger) *Source { + return &Source{ + frostFS: frostFS, + netmapCache: netmapCache, + bucketCache: bucketCache, + log: log, + } +} + +func (s *Source) NetMapSnapshot(ctx context.Context) (netmap.NetMap, error) { + cachedNetmap := s.netmapCache.Get() + if cachedNetmap != nil { + return *cachedNetmap, nil + } + + netmapSnapshot, err := s.frostFS.NetmapSnapshot(ctx) + if err != nil { + return netmap.NetMap{}, fmt.Errorf("get netmap: %w", err) + } + + if err = s.netmapCache.Put(netmapSnapshot); err != nil { + s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err)) + } + + return netmapSnapshot, nil +} + +func (s *Source) PlacementPolicy(ctx context.Context, cnrID cid.ID) (netmap.PlacementPolicy, error) { + info := s.bucketCache.GetByCID(cnrID) + if info != nil { + return info.PlacementPolicy, nil + } + + prm := handler.PrmContainer{ + ContainerID: cnrID, + } + res, err := s.frostFS.Container(ctx, prm) + if err != nil { + return netmap.PlacementPolicy{}, fmt.Errorf("get container: %w", err) + } + + // We don't put container back to the cache to keep cache + // coherent to the requests made by users. FrostFS Source + // is being used by SDK Tree Pool and it should not fill cache + // with possibly irrelevant container values. + + return res.PlacementPolicy(), nil +} From f0c999d9a28e9b6cf3419c1147178ce8626f4c6e Mon Sep 17 00:00:00 2001 From: Aleksey Kravchenko Date: Tue, 24 Dec 2024 18:42:02 +0300 Subject: [PATCH 38/86] [#188] Improve content-type detector Signed-off-by: Aleksey Kravchenko --- internal/handler/head.go | 15 ++++++- internal/handler/reader.go | 21 ++++++++-- internal/handler/reader_test.go | 71 ++++++++++++++++++++++++++------- 3 files changed, 87 insertions(+), 20 deletions(-) diff --git a/internal/handler/head.go b/internal/handler/head.go index f2e9f38..da96eff 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -45,7 +45,11 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid } req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10)) - var contentType string + var ( + contentType string + filename string + filepath string + ) for _, attr := range obj.Attributes() { key := attr.Key() val := attr.Value() @@ -69,8 +73,15 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid req.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val + case object.AttributeFilePath: + filepath = val + case object.AttributeFileName: + filename = val } } + if filename == "" { + filename = filepath + } idsToResponse(&req.Response, obj) @@ -85,7 +96,7 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid } return h.frostfs.RangeObject(ctx, prmRange) - }) + }, filename) if err != nil && err != io.EOF { req.handleFrostFSErr(err, start) return diff --git a/internal/handler/reader.go b/internal/handler/reader.go index 50121c9..60067ab 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -4,9 +4,11 @@ import ( "bytes" "context" "io" + "mime" "net/http" "path" "strconv" + "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" @@ -25,7 +27,7 @@ type readCloser struct { // initializes io.Reader with the limited size and detects Content-Type from it. // Returns r's error directly. Also returns the processed data. -func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (string, []byte, error) { +func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error), filename string) (string, []byte, error) { if maxSize > sizeToDetectType { maxSize = sizeToDetectType } @@ -44,7 +46,20 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (str buf = buf[:n] - return http.DetectContentType(buf), buf, err // to not lose io.EOF + contentType := http.DetectContentType(buf) + + // Since the detector detects the "text/plain" content type for various types of text files, + // including CSS, JavaScript, and CSV files, + // we'll determine the final content type based on the file's extension. + if strings.HasPrefix(contentType, "text/plain") { + ext := path.Ext(filename) + // If the file doesn't have a file extension, we'll keep the content type as is. + if len(ext) > 0 { + contentType = mime.TypeByExtension(ext) + } + } + + return contentType, buf, err // to not lose io.EOF } type getMultiobjectBodyParams struct { @@ -128,7 +143,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) { return payload, nil - }) + }, filename) if err != nil && err != io.EOF { req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) response.Error(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) diff --git a/internal/handler/reader_test.go b/internal/handler/reader_test.go index c63a734..f867677 100644 --- a/internal/handler/reader_test.go +++ b/internal/handler/reader_test.go @@ -10,39 +10,80 @@ import ( "github.com/stretchr/testify/require" ) +const ( + txtContentType = "text/plain; charset=utf-8" + cssContentType = "text/css; charset=utf-8" + htmlContentType = "text/html; charset=utf-8" + javascriptContentType = "text/javascript; charset=utf-8" + + htmlBody = "Test Html" +) + func TestDetector(t *testing.T) { - txtContentType := "text/plain; charset=utf-8" sb := strings.Builder{} for i := 0; i < 10; i++ { sb.WriteString("Some txt content. Content-Type must be detected properly by detector.") } for _, tc := range []struct { - Name string - ContentType string - Expected string + Name string + ExpectedContentType string + Content string + FileName string }{ { - Name: "less than 512b", - ContentType: txtContentType, - Expected: sb.String()[:256], + Name: "less than 512b", + ExpectedContentType: txtContentType, + Content: sb.String()[:256], + FileName: "test.txt", }, { - Name: "more than 512b", - ContentType: txtContentType, - Expected: sb.String(), + Name: "more than 512b", + ExpectedContentType: txtContentType, + Content: sb.String(), + FileName: "test.txt", + }, + { + Name: "css content type", + ExpectedContentType: cssContentType, + Content: sb.String(), + FileName: "test.css", + }, + { + Name: "javascript content type", + ExpectedContentType: javascriptContentType, + Content: sb.String(), + FileName: "test.js", + }, + { + Name: "html content type by file content", + ExpectedContentType: htmlContentType, + Content: htmlBody, + FileName: "test.detect-by-content", + }, + { + Name: "html content type by file extension", + ExpectedContentType: htmlContentType, + Content: sb.String(), + FileName: "test.html", + }, + { + Name: "empty file extension", + ExpectedContentType: txtContentType, + Content: sb.String(), + FileName: "test", }, } { t.Run(tc.Name, func(t *testing.T) { - contentType, data, err := readContentType(uint64(len(tc.Expected)), + contentType, data, err := readContentType(uint64(len(tc.Content)), func(uint64) (io.Reader, error) { - return strings.NewReader(tc.Expected), nil - }, + return strings.NewReader(tc.Content), nil + }, tc.FileName, ) require.NoError(t, err) - require.Equal(t, tc.ContentType, contentType) - require.True(t, strings.HasPrefix(tc.Expected, string(data))) + require.Equal(t, tc.ExpectedContentType, contentType) + require.True(t, strings.HasPrefix(tc.Content, string(data))) }) } } From 4b782cf1247fcf2cac42d1c58a610c4c98d8698e Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 9 Jan 2025 12:28:38 +0300 Subject: [PATCH 39/86] [#187] Add handling quota limit reached error The Access Denied status may be received from APE due to exceeding the quota. In this situation, you need to return the appropriate status code. Signed-off-by: Roman Loginov --- CHANGELOG.md | 3 + cmd/http-gw/app.go | 21 +++--- internal/handler/download.go | 5 +- internal/handler/handler.go | 13 ++-- internal/handler/reader.go | 3 +- internal/handler/upload.go | 13 ++-- internal/handler/utils.go | 45 +++++++++++-- internal/service/frostfs/frostfs.go | 4 ++ internal/service/frostfs/frostfs_test.go | 83 ++++++++++++++++++++++++ response/utils.go | 41 ------------ 10 files changed, 156 insertions(+), 75 deletions(-) create mode 100644 internal/service/frostfs/frostfs_test.go delete mode 100644 response/utils.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e528b8b..fd37815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Added +- Add handling quota limit reached error (#187) + ## [0.32.0] - Khumbu - 2024-12-20 ### Fixed diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 3386536..e34386c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -25,7 +25,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -636,28 +635,28 @@ func (a *app) stopServices() { } } -func (a *app) configureRouter(handler *handler.Handler) { +func (a *app) configureRouter(h *handler.Handler) { r := router.New() r.RedirectTrailingSlash = true r.NotFound = func(r *fasthttp.RequestCtx) { - response.Error(r, "Not found", fasthttp.StatusNotFound) + handler.ResponseError(r, "Not found", fasthttp.StatusNotFound) } r.MethodNotAllowed = func(r *fasthttp.RequestCtx) { - response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) + handler.ResponseError(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.addMiddlewares(handler.Upload)) + r.POST("/upload/{cid}", a.addMiddlewares(h.Upload)) r.OPTIONS("/upload/{cid}", a.addPreflight()) a.log.Info(logs.AddedPathUploadCid) - r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(handler.DownloadByAddressOrBucketName)) - r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(handler.HeadByAddressOrBucketName)) + r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(h.DownloadByAddressOrBucketName)) + r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(h.HeadByAddressOrBucketName)) r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetCidOid) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(handler.DownloadByAttribute)) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(handler.HeadByAttribute)) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.DownloadByAttribute)) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute)) r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(handler.DownloadZipped)) + r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZipped)) r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) a.log.Info(logs.AddedPathZipCidPrefix) @@ -800,7 +799,7 @@ func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { log := utils.GetReqLogOrDefault(reqCtx, a.log) log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err)) - response.Error(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) + handler.ResponseError(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } utils.SetContextToRequest(appCtx, req) diff --git a/internal/handler/download.go b/internal/handler/download.go index de27fa3..8766f0c 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -120,7 +119,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { prefix, err := url.QueryUnescape(prefix) if err != nil { log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) - response.Error(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -135,7 +134,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) - response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 1150f45..2f1c6ad 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -140,6 +139,8 @@ var ( ErrAccessDenied = errors.New("access denied") // ErrGatewayTimeout is returned from FrostFS in case of timeout, deadline exceeded etc. ErrGatewayTimeout = errors.New("gateway timeout") + // ErrQuotaLimitReached is returned from FrostFS in case of quota exceeded. + ErrQuotaLimitReached = errors.New("quota limit reached") ) // FrostFS represents virtual connection to FrostFS network. @@ -210,7 +211,7 @@ func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path } if foundOID.IsDeleteMarker { log.Error(logs.ObjectWasDeleted) - response.Error(c, "object deleted", fasthttp.StatusNotFound) + ResponseError(c, "object deleted", fasthttp.StatusNotFound) return } @@ -230,14 +231,14 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte key, err := url.QueryUnescape(key) if err != nil { log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err)) - response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) return } val, err = url.QueryUnescape(val) if err != nil { log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err)) - response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -252,11 +253,11 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte objID, err := h.findObjectByAttribute(ctx, log, bktInfo.CID, key, val) if err != nil { if errors.Is(err, io.EOF) { - response.Error(c, err.Error(), fasthttp.StatusNotFound) + ResponseError(c, err.Error(), fasthttp.StatusNotFound) return } - response.Error(c, err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/reader.go b/internal/handler/reader.go index 60067ab..cbd8294 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -12,7 +12,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -146,7 +145,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A }, filename) if err != nil && err != io.EOF { req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) - response.Error(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 867025d..9493635 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -9,7 +9,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" @@ -81,14 +80,14 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) - response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } filtered, err := filterHeaders(log, &c.Request.Header) if err != nil { log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) - response.Error(c, err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } @@ -103,7 +102,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { if err = utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) - response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -157,7 +156,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) - response.Error(c, "could not encode response", fasthttp.StatusBadRequest) + ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) return } @@ -179,11 +178,11 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { } func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *zap.Logger) { - statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err) + statusCode, msg, additionalFields := formErrorResponse("could not store file in frostfs", err) logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) - response.Error(r, msg, statusCode) + ResponseError(r, msg, statusCode) } func (h *Handler) fetchBearerToken(ctx context.Context) *bearer.Token { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index d09ed23..7fdd396 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -2,14 +2,16 @@ package handler import ( "context" + "errors" + "fmt" "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -27,11 +29,11 @@ func (r *request) handleFrostFSErr(err error, start time.Time) { zap.Stringer("elapsed", time.Since(start)), zap.Error(err), } - statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err) + statusCode, msg, additionalFields := formErrorResponse("could not receive object", err) logFields = append(logFields, additionalFields...) r.log.Error(logs.CouldNotReceiveObject, logFields...) - response.Error(r.RequestCtx, msg, statusCode) + ResponseError(r.RequestCtx, msg, statusCode) } func bearerToken(ctx context.Context) *bearer.Token { @@ -79,10 +81,10 @@ func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { log.Error(logs.CouldntGetBucket, zap.Error(err)) if client.IsErrContainerNotFound(err) { - response.Error(c, "Not Found", fasthttp.StatusNotFound) + ResponseError(c, "Not Found", fasthttp.StatusNotFound) return } - response.Error(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) } func newAddress(cnr cid.ID, obj oid.ID) oid.Address { @@ -91,3 +93,36 @@ func newAddress(cnr cid.ID, obj oid.ID) oid.Address { addr.SetObject(obj) return addr } + +func ResponseError(r *fasthttp.RequestCtx, msg string, code int) { + r.Error(msg+"\n", code) +} + +func formErrorResponse(message string, err error) (int, string, []zap.Field) { + var ( + msg string + statusCode int + logFields []zap.Field + ) + + st := new(sdkstatus.ObjectAccessDenied) + + switch { + case errors.As(err, &st): + statusCode = fasthttp.StatusForbidden + reason := st.Reason() + msg = fmt.Sprintf("%s: %v: %s", message, err, reason) + logFields = append(logFields, zap.String("error_detail", reason)) + case errors.Is(err, ErrQuotaLimitReached): + statusCode = fasthttp.StatusConflict + msg = fmt.Sprintf("%s: %v", message, err) + case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err): + statusCode = fasthttp.StatusNotFound + msg = "Not Found" + default: + statusCode = fasthttp.StatusBadRequest + msg = fmt.Sprintf("%s: %v", message, err) + } + + return statusCode, msg, logFields +} diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index b218976..c6af526 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -215,6 +215,10 @@ func handleObjectError(msg string, err error) error { } if reason, ok := IsErrObjectAccessDenied(err); ok { + if strings.Contains(reason, "limit reached") { + return fmt.Errorf("%s: %w: %s", msg, handler.ErrQuotaLimitReached, reason) + } + return fmt.Errorf("%s: %w: %s", msg, handler.ErrAccessDenied, reason) } diff --git a/internal/service/frostfs/frostfs_test.go b/internal/service/frostfs/frostfs_test.go new file mode 100644 index 0000000..e9b3329 --- /dev/null +++ b/internal/service/frostfs/frostfs_test.go @@ -0,0 +1,83 @@ +package frostfs + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestHandleObjectError(t *testing.T) { + msg := "some msg" + + t.Run("nil error", func(t *testing.T) { + err := handleObjectError(msg, nil) + require.Nil(t, err) + }) + + t.Run("simple access denied", func(t *testing.T) { + reason := "some reason" + inputErr := new(apistatus.ObjectAccessDenied) + inputErr.WriteReason(reason) + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrAccessDenied) + require.Contains(t, err.Error(), reason) + require.Contains(t, err.Error(), msg) + }) + + t.Run("access denied - quota reached", func(t *testing.T) { + reason := "Quota limit reached" + inputErr := new(apistatus.ObjectAccessDenied) + inputErr.WriteReason(reason) + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrQuotaLimitReached) + require.Contains(t, err.Error(), reason) + require.Contains(t, err.Error(), msg) + }) + + t.Run("simple timeout", func(t *testing.T) { + inputErr := errors.New("timeout") + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrGatewayTimeout) + require.Contains(t, err.Error(), inputErr.Error()) + require.Contains(t, err.Error(), msg) + }) + + t.Run("deadline exceeded", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + <-ctx.Done() + + err := handleObjectError(msg, ctx.Err()) + require.ErrorIs(t, err, handler.ErrGatewayTimeout) + require.Contains(t, err.Error(), ctx.Err().Error()) + require.Contains(t, err.Error(), msg) + }) + + t.Run("grpc deadline exceeded", func(t *testing.T) { + inputErr := fmt.Errorf("wrap grpc error: %w", status.Error(codes.DeadlineExceeded, "error")) + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrGatewayTimeout) + require.Contains(t, err.Error(), inputErr.Error()) + require.Contains(t, err.Error(), msg) + }) + + t.Run("unknown error", func(t *testing.T) { + inputErr := errors.New("unknown error") + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, inputErr) + require.Contains(t, err.Error(), msg) + }) +} diff --git a/response/utils.go b/response/utils.go deleted file mode 100644 index f233943..0000000 --- a/response/utils.go +++ /dev/null @@ -1,41 +0,0 @@ -package response - -import ( - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "github.com/valyala/fasthttp" - "go.uber.org/zap" -) - -func Error(r *fasthttp.RequestCtx, msg string, code int) { - r.Error(msg+"\n", code) -} - -func FormErrorResponse(message string, err error) (int, string, []zap.Field) { - var ( - msg string - statusCode int - logFields []zap.Field - ) - - st := new(sdkstatus.ObjectAccessDenied) - - switch { - case errors.As(err, &st): - statusCode = fasthttp.StatusForbidden - reason := st.Reason() - msg = fmt.Sprintf("%s: %v: %s", message, err, reason) - logFields = append(logFields, zap.String("error_detail", reason)) - case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err): - statusCode = fasthttp.StatusNotFound - msg = "Not Found" - default: - statusCode = fasthttp.StatusBadRequest - msg = fmt.Sprintf("%s: %v", message, err) - } - - return statusCode, msg, logFields -} From 1e82f64dfde297712006b1fea80fd186c1d0d455 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 21 Jan 2025 11:07:00 +0300 Subject: [PATCH 40/86] [#193] Enable integration tests in Forgejo Actions Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/tests.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 81d93dc..d4182ed 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -43,3 +43,19 @@ jobs: - name: Run tests run: make test + + integration: + name: Integration tests + runs-on: oci-runner + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.23' + + - name: Run integration tests + run: |- + podman-service.sh + make integration-test From 856e0ecf40fd659b5f9ebd182888cfa7a269bf26 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 21 Jan 2025 11:33:58 +0300 Subject: [PATCH 41/86] [#193] Update testcontainers to v0.35.0 Signed-off-by: Vitaliy Potyarkin --- cmd/http-gw/integration_test.go | 15 +- go.mod | 60 ++- go.sum | 819 ++++---------------------------- 3 files changed, 135 insertions(+), 759 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 4c20546..0c2bdf4 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -29,6 +29,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + docker "github.com/docker/docker/api/types/container" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/viper" @@ -434,11 +435,13 @@ func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, o func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container { req := testcontainers.ContainerRequest{ - Image: image, - WaitingFor: wait.NewLogStrategy("aio container started").WithStartupTimeout(2 * time.Minute), - Name: "aio", - Hostname: "aio", - NetworkMode: "host", + Image: image, + WaitingFor: wait.NewLogStrategy("aio container started").WithStartupTimeout(2 * time.Minute), + Name: "aio", + Hostname: "aio", + HostConfigModifier: func(hc *docker.HostConfig) { + hc.NetworkMode = "host" + }, } aioC, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, @@ -539,7 +542,7 @@ func registerUser(t *testing.T, ctx context.Context, aioContainer testcontainers err := aioContainer.CopyFileToContainer(ctx, pathToWallet, "/usr/wallet.json", 644) require.NoError(t, err) - _, err = aioContainer.Exec(ctx, []string{ + _, _, err = aioContainer.Exec(ctx, []string{ "/usr/bin/frostfs-s3-authmate", "register-user", "--wallet", "/usr/wallet.json", "--rpc-endpoint", "http://localhost:30333", diff --git a/go.mod b/go.mod index 0b74841..46fe3bc 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,8 @@ require ( git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 - github.com/docker/go-units v0.4.0 + github.com/docker/docker v27.1.1+incompatible + github.com/docker/go-units v0.5.0 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.106.2 github.com/panjf2000/ants/v2 v2.5.0 @@ -18,7 +19,7 @@ require ( github.com/spf13/viper v1.15.0 github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 - github.com/testcontainers/testcontainers-go v0.13.0 + github.com/testcontainers/testcontainers-go v0.35.0 github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/valyala/fasthttp v1.34.0 go.opentelemetry.io/otel v1.28.0 @@ -26,56 +27,60 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.26.0 - golang.org/x/sys v0.22.0 + golang.org/x/sys v0.28.0 google.golang.org/grpc v1.66.2 ) require ( + dario.cat/mergo v1.0.0 // indirect git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e // indirect git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/Microsoft/hcsshim v0.9.2 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/VictoriaMetrics/easyproto v0.1.4 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/cgroups v1.0.3 // indirect - github.com/containerd/containerd v1.6.2 // indirect + github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/docker v20.10.14+incompatible // indirect - github.com/docker/go-connections v0.4.0 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/ipfs/go-cid v0.0.7 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.16.4 // indirect + github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/sys/mount v0.3.2 // indirect - github.com/moby/sys/mountinfo v0.6.1 // indirect - github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect @@ -88,27 +93,32 @@ require ( github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.2 // indirect - github.com/opencontainers/runc v1.1.1 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect - github.com/sirupsen/logrus v1.8.1 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/twmb/murmur3 v1.1.8 // indirect - github.com/urfave/cli v1.22.5 // indirect + github.com/urfave/cli v1.22.12 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.9 // indirect - go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect @@ -116,10 +126,10 @@ require ( go.opentelemetry.io/otel/sdk v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/term v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect diff --git a/go.sum b/go.sum index 06bdd7d..e5bfc09 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -36,6 +35,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= @@ -55,357 +56,101 @@ git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjq git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 h1:HeY8n27VyPRQe49l/fzyVMkWEB2fsLJYKp64pwA7tz4= git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw= -github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= -github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= -github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= -github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= -github.com/Microsoft/hcsshim v0.9.2 h1:wB06W5aYFfUB3IvootYAY2WnOmIdgPGfqSI6tufQNnY= -github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= -github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/VictoriaMetrics/easyproto v0.1.4 h1:r8cNvo8o6sR4QShBXQd1bKw/VVLSQma/V2KhTBPf+Sc= github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= -github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= -github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb h1:f0BMgIjhZy4lSRHCXFbQst85f5agZAjtDMixQqBWNpc= github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= -github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= -github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= -github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= -github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= -github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBdAP4= -github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= -github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= -github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= -github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= -github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= -github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= -github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= -github.com/containerd/containerd v1.6.2 h1:pcaPUGbYW8kBw6OgIZwIVIeEhdWVrBzsoCfVJ5BjrLU= -github.com/containerd/containerd v1.6.2/go.mod h1:sidY30/InSE1j2vdD1ihtKoJz+lWdaXMdiAeIupaf+s= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= -github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= -github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= -github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/continuity v0.2.2 h1:QSqfxcn8c+12slxwu00AtzXrsami0MJb/MQs9lOLHLA= -github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= -github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= -github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= -github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= -github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= -github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= -github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= -github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= -github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= -github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= -github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= -github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= -github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= -github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= -github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= -github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= -github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= -github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= -github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= -github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= -github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.14+incompatible h1:+T9/PRYWNDo5SZl5qS1r9Mo/0Q8AwxKKPtu9S1yxM0w= -github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp/router v1.4.1 h1:3xPUO+hy/HAkgGDSd5sX5w18cyGDIFbC7vip8KwPDk8= github.com/fasthttp/router v1.4.1/go.mod h1:4P0Kq4C882tA2evBKDW7De7hGfWmvV8FN+zqt8Lu49Q= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= -github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= -github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= -github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -413,7 +158,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -428,8 +172,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -443,15 +185,11 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -466,42 +204,17 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -513,116 +226,58 @@ github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXei github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= -github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= -github.com/moby/sys/mount v0.3.2 h1:uq/CiGDZPvr+c85RYHtKIUORFbmavBUyWH3E1NEyjqI= -github.com/moby/sys/mount v0.3.2/go.mod h1:iN27Ec0LtJ0Mx/++rE6t6mTdbbEEZd+oKfAHP1y6vHs= -github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/mountinfo v0.6.1 h1:+H/KnGEAGRpTrEAqNVQ2AM3SiwMgJUt/TXj+Z8cmCIc= -github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= @@ -640,12 +295,6 @@ github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsC github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc= github.com/nspcc-dev/neo-go v0.106.2 h1:KXSJ2J5Oacc7LrX3r4jvnC8ihKqHs5NB21q4f2S3r9o= @@ -656,256 +305,127 @@ github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/Rg github.com/nspcc-dev/rfc6979 v0.2.1/go.mod h1:Tk7h5kyUWkhjyO3zUgFFhy1v2vQv3BvQEntakdtqrWc= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= -github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runc v1.1.1 h1:PJ9DSs2sVwE0iVr++pAHE6QkS9tzcVWozlPifdwMgrU= -github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= -github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= -github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q= github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 h1:N3Af8f13ooDKcIhsmFT7Z05CStZWu4C7Md0uDEy4q6o= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873/go.mod h1:dmPawKuiAeG/aFYVs2i+Dyosoo7FNcm+Pi8iK6ZUrX8= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= -github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/ssgreg/journald v1.0.0 h1:0YmTDPJXxcWDPba12qNMdO6TxvfkFSYpFIJ31CwmLcU= github.com/ssgreg/journald v1.0.0/go.mod h1:RUckwmTM8ghGWPslq2+ZBZzbb9/2KgjzYZ4JEP+oRt0= -github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= -github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBNGN0TYb/7oKIPVn15JA= -github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/testcontainers/testcontainers-go v0.35.0 h1:uADsZpTKFAtp8SLK+hMwSaa+X+JiERHtd4sQAFmXeMo= +github.com/testcontainers/testcontainers-go v0.35.0/go.mod h1:oEVBj5zrfJTrgjwONs1SsRbnBtH9OKl+IGl3UMcr2B4= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 h1:GpfJ7OdNjS7BFTVwNCUI9L4aCJOFRbr5fdHqjdhoYE8= github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4/go.mod h1:f3jBhpWvuZmue0HZK52GzRHJOYHYSILs/c8+K2S/J+o= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= -github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA= github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= @@ -914,39 +434,26 @@ go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBq go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -982,31 +489,21 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1023,17 +520,12 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= @@ -1057,52 +549,29 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1112,51 +581,36 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1165,35 +619,25 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1216,17 +660,14 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1234,8 +675,6 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1244,7 +683,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1271,13 +709,11 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1286,7 +722,6 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1297,17 +732,14 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1317,14 +749,10 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1: google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1334,12 +762,9 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1352,41 +777,20 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -1394,12 +798,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/gotestsum v1.7.0/go.mod h1:V1m4Jw3eBerhI/A6qCxUE07RnCg7ACkKj9BYcAm09V8= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1407,36 +807,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= -k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= -k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= -k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= -k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= -k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -1444,10 +814,3 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From a7617514d39c2bd06ae006a3e28a3471a6bcecb0 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 21 Jan 2025 12:59:25 +0300 Subject: [PATCH 42/86] [#193] Use selfhosted image registry instead of Docker Hub Existing AIO image tags referenced from our integration tests were manually synced to git.frostfs.info prior to this change. Signed-off-by: Vitaliy Potyarkin --- cmd/http-gw/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 0c2bdf4..c3c5de5 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -51,7 +51,7 @@ const ( func TestIntegration(t *testing.T) { rootCtx := context.Background() - aioImage := "truecloudlab/frostfs-aio:" + aioImage := "git.frostfs.info/truecloudlab/frostfs-aio:" versions := []string{ "1.2.7", "1.3.0", From 7901d00924e93f387766c1400195fb5a56c65abf Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 13 Dec 2024 16:00:31 +0300 Subject: [PATCH 43/86] [#170] Support tar.gz downloading Split DownloadZip handler on methods. Add handler DownloadTar for downloading tar.gz archives. Make methods more universal for using in both implementations Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 14 +- cmd/http-gw/settings.go | 15 +- config/config.env | 6 +- config/config.yaml | 10 +- docs/gate-configuration.md | 14 +- internal/handler/download.go | 228 +++++++++++++++++--------- internal/handler/handler.go | 4 +- internal/handler/handler_fuzz_test.go | 2 +- internal/handler/handler_test.go | 2 +- internal/handler/head.go | 2 +- internal/handler/utils.go | 7 + internal/logs/logs.go | 4 +- 12 files changed, 214 insertions(+), 94 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index e34386c..56c6be1 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -97,7 +97,7 @@ type ( mu sync.RWMutex defaultTimestamp bool - zipCompression bool + archiveCompression bool clientCut bool returnIndexPage bool indexPageTemplate string @@ -178,7 +178,7 @@ func (a *app) initAppSettings() { func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { defaultTimestamp := v.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) - zipCompression := v.GetBool(cfgZipCompression) + archiveCompression := fetchArchiveCompression(v) returnIndexPage := v.GetBool(cfgIndexPageEnabled) clientCut := v.GetBool(cfgClientCut) bufferMaxSizeForPut := v.GetUint64(cfgBufferMaxSizeForPut) @@ -197,7 +197,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { defer s.mu.Unlock() s.defaultTimestamp = defaultTimestamp - s.zipCompression = zipCompression + s.archiveCompression = archiveCompression s.returnIndexPage = returnIndexPage s.clientCut = clientCut s.bufferMaxSizeForPut = bufferMaxSizeForPut @@ -236,10 +236,10 @@ func (s *appSettings) DefaultTimestamp() bool { return s.defaultTimestamp } -func (s *appSettings) ZipCompression() bool { +func (s *appSettings) ArchiveCompression() bool { s.mu.RLock() defer s.mu.RUnlock() - return s.zipCompression + return s.archiveCompression } func (s *appSettings) IndexPageEnabled() bool { @@ -656,8 +656,10 @@ func (a *app) configureRouter(h *handler.Handler) { r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute)) r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZipped)) + r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZip)) r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) + r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadTar)) + r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight()) a.log.Info(logs.AddedPathZipCidPrefix) a.webServer.Handler = r.Handler diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 691e9ba..5cf06a0 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -128,8 +128,13 @@ const ( cfgResolveOrder = "resolve_order" // Zip compression. + // + // Deprecated: Use cfgArchiveCompression instead. cfgZipCompression = "zip.compression" + // Archive compression. + cfgArchiveCompression = "archive.compression" + // Runtime. cfgSoftMemoryLimit = "runtime.soft_memory_limit" @@ -255,9 +260,6 @@ func settings() *viper.Viper { // upload header v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) - // zip: - v.SetDefault(cfgZipCompression, false) - // metrics v.SetDefault(cfgPprofAddress, "localhost:8083") v.SetDefault(cfgPrometheusAddress, "localhost:8084") @@ -844,3 +846,10 @@ func fetchTracingAttributes(v *viper.Viper) (map[string]string, error) { return attributes, nil } + +func fetchArchiveCompression(v *viper.Viper) bool { + if v.IsSet(cfgZipCompression) { + return v.GetBool(cfgZipCompression) + } + return v.GetBool(cfgArchiveCompression) +} diff --git a/config/config.env b/config/config.env index 171889f..db619b5 100644 --- a/config/config.env +++ b/config/config.env @@ -97,9 +97,13 @@ HTTP_GW_REBALANCE_TIMER=30s # The number of errors on connection after which node is considered as unhealthy HTTP_GW_POOL_ERROR_THRESHOLD=100 -# Enable zip compression to download files by common prefix. +# Enable archive compression to download files by common prefix. +# DEPRECATED: Use HTTP_GW_ARCHIVE_COMPRESSION instead. HTTP_GW_ZIP_COMPRESSION=false +# Enable archive compression to download files by common prefix. +HTTP_GW_ARCHIVE_COMPRESSION=false + HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_EXPORTER="otlp_grpc" diff --git a/config/config.yaml b/config/config.yaml index eee84e5..a70ec9a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -116,13 +116,19 @@ pool_error_threshold: 100 # The number of errors on connection after which node # Number of workers in handler's worker pool worker_pool_size: 1000 -# Enable index page to see objects list for specified container and prefix +# Enables index page to see objects list for specified container and prefix index_page: enabled: false template_path: internal/handler/templates/index.gotmpl +# Deprecated: Use archive.compression instead zip: - compression: false # Enable zip compression to download files by common prefix. + # Enables zip compression to download files by common prefix. + compression: false + +archive: + # Enables archive compression to download files by common prefix. + compression: false runtime: soft_memory_limit: 1gb diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index ce7c0c7..6aadd1f 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -218,9 +218,10 @@ upload_header: |-------------------------|--------|---------------|---------------|-------------------------------------------------------------| | `use_default_timestamp` | `bool` | yes | `false` | Create timestamp for object if it isn't provided by header. | - # `zip` section +> **_DEPRECATED:_** Use archive section instead + ```yaml zip: compression: false @@ -230,6 +231,17 @@ zip: |---------------|--------|---------------|---------------|--------------------------------------------------------------| | `compression` | `bool` | yes | `false` | Enable zip compression when download files by common prefix. | +# `archive` section + +```yaml +archive: + compression: false +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------|--------|---------------|---------------|------------------------------------------------------------------| +| `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. | + # `pprof` section diff --git a/internal/handler/download.go b/internal/handler/download.go index 8766f0c..684e3b8 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -1,20 +1,21 @@ package handler import ( + "archive/tar" "archive/zip" "bufio" + "compress/gzip" "context" "errors" "fmt" "io" - "net/http" "net/url" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -46,7 +47,7 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { return } - req := h.newRequest(c, log) + req := newRequest(c, log) var objID oid.ID if checkS3Err == nil && shouldDownload(oidParam, downloadParam) { @@ -62,13 +63,6 @@ func shouldDownload(oidParam string, downloadParam bool) bool { return !isDir(oidParam) || downloadParam } -func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) request { - return request{ - RequestCtx: ctx, - log: log, - } -} - // DownloadByAttribute handles attribute-based download requests. func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { h.byAttribute(c, h.receiveFile) @@ -90,13 +84,61 @@ func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op return h.frostfs.SearchObjects(ctx, prm) } -func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { +// DownloadZip handles zip by prefix requests. +func (h *Handler) DownloadZip(c *fasthttp.RequestCtx) { + scid, _ := c.UserValue("cid").(string) + + ctx := utils.GetContextFromRequest(c) + log := utils.GetReqLogOrDefault(ctx, h.log) + bktInfo, err := h.getBucketInfo(ctx, scid, log) + if err != nil { + logAndSendBucketError(c, log, err) + return + } + resSearch, err := h.searchObjectsByPrefix(c, log, bktInfo.CID) + if err != nil { + return + } + + c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") + c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") + + c.SetBodyStreamWriter(h.getZipResponseWriter(ctx, log, resSearch, bktInfo)) +} + +func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { + return func(w *bufio.Writer) { + defer resSearch.Close() + + buf := make([]byte, 3<<20) + zipWriter := zip.NewWriter(w) + var objectsWritten int + + errIter := resSearch.Iterate(h.putObjectToArchive(ctx, log, bktInfo.CID, buf, + func(obj *object.Object) (io.Writer, error) { + objectsWritten++ + return h.createZipFile(zipWriter, obj) + }), + ) + if errIter != nil { + log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) + return + } else if objectsWritten == 0 { + log.Warn(logs.ObjectsNotFound) + } + if err := zipWriter.Close(); err != nil { + log.Error(logs.CloseZipWriter, zap.Error(err)) + } + } +} + +func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, error) { method := zip.Store if h.config.ZipCompression() { method = zip.Deflate } - filePath := getZipFilePath(obj) + filePath := getFilePath(obj) if len(filePath) == 0 || filePath[len(filePath)-1] == '/' { return nil, fmt.Errorf("invalid filepath '%s'", filePath) } @@ -108,99 +150,139 @@ func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, }) } -// DownloadZipped handles zip by prefix requests. -func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { +// DownloadTar forms tar.gz from objects by prefix. +func (h *Handler) DownloadTar(c *fasthttp.RequestCtx) { scid, _ := c.UserValue("cid").(string) - prefix, _ := c.UserValue("prefix").(string) ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log) - - prefix, err := url.QueryUnescape(prefix) - if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) - ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) - return - } - - log = log.With(zap.String("cid", scid), zap.String("prefix", prefix)) - bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { logAndSendBucketError(c, log, err) return } - - resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) + resSearch, err := h.searchObjectsByPrefix(c, log, bktInfo.CID) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) - ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return } - c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") - c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") - c.Response.SetStatusCode(http.StatusOK) + c.Response.Header.Set(fasthttp.HeaderContentType, "application/gzip") + c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.tar.gz\"") - c.SetBodyStreamWriter(func(w *bufio.Writer) { + c.SetBodyStreamWriter(h.getTarResponseWriter(ctx, log, resSearch, bktInfo)) +} + +func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { + return func(w *bufio.Writer) { defer resSearch.Close() - zipWriter := zip.NewWriter(w) + compressionLevel := gzip.NoCompression + if h.config.ZipCompression() { + compressionLevel = gzip.DefaultCompression + } - var bufZip []byte - var addr oid.Address + // ignore error because it's not nil only if compressionLevel argument is invalid + gzipWriter, _ := gzip.NewWriterLevel(w, compressionLevel) + tarWriter := tar.NewWriter(gzipWriter) - empty := true - called := false - btoken := bearerToken(ctx) - addr.SetContainer(bktInfo.CID) - - errIter := resSearch.Iterate(func(id oid.ID) bool { - called = true - - if empty { - bufZip = make([]byte, 3<<20) // the same as for upload + defer func() { + if err := tarWriter.Close(); err != nil { + log.Error(logs.CloseTarWriter, zap.Error(err)) } - empty = false - - addr.SetObject(id) - if err = h.zipObject(ctx, zipWriter, addr, btoken, bufZip); err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.String("oid", id.EncodeToString()), zap.Error(err)) + if err := gzipWriter.Close(); err != nil { + log.Error(logs.CloseGzipWriter, zap.Error(err)) } + }() - return false - }) + var objectsWritten int + buf := make([]byte, 3<<20) // the same as for upload + + errIter := resSearch.Iterate(h.putObjectToArchive(ctx, log, bktInfo.CID, buf, + func(obj *object.Object) (io.Writer, error) { + objectsWritten++ + return h.createTarFile(tarWriter, obj) + }), + ) if errIter != nil { log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) - } else if !called { - log.Error(logs.ObjectsNotFound) + } else if objectsWritten == 0 { + log.Warn(logs.ObjectsNotFound) } + } +} - if err = zipWriter.Close(); err != nil { - log.Error(logs.CloseZipWriter, zap.Error(err)) - } +func (h *Handler) createTarFile(tw *tar.Writer, obj *object.Object) (io.Writer, error) { + filePath := getFilePath(obj) + if len(filePath) == 0 || filePath[len(filePath)-1] == '/' { + return nil, fmt.Errorf("invalid filepath '%s'", filePath) + } + + return tw, tw.WriteHeader(&tar.Header{ + Name: filePath, + Mode: 0655, + Size: int64(obj.PayloadSize()), }) } -func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error { - prm := PrmObjectGet{ - PrmAuth: PrmAuth{ - BearerToken: btoken, - }, - Address: addr, - } +func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID cid.ID, buf []byte, createArchiveHeader func(obj *object.Object) (io.Writer, error)) func(id oid.ID) bool { + return func(id oid.ID) bool { + log = log.With(zap.String("oid", id.EncodeToString())) - resGet, err := h.frostfs.GetObject(ctx, prm) + prm := PrmObjectGet{ + PrmAuth: PrmAuth{ + BearerToken: bearerToken(ctx), + }, + Address: newAddress(cnrID, id), + } + + resGet, err := h.frostfs.GetObject(ctx, prm) + if err != nil { + log.Error(logs.FailedToGetObject, zap.Error(err)) + return false + } + + fileWriter, err := createArchiveHeader(&resGet.Header) + if err != nil { + log.Error(logs.FailedToAddObjectToArchive, zap.Error(err)) + return false + } + + if err = writeToArchive(resGet, fileWriter, buf); err != nil { + log.Error(logs.FailedToAddObjectToArchive, zap.Error(err)) + return false + } + + return false + } +} + +func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, cnrID cid.ID) (ResObjectSearch, error) { + scid := cnrID.EncodeToString() + prefix, _ := c.UserValue("prefix").(string) + + ctx := utils.GetContextFromRequest(c) + + prefix, err := url.QueryUnescape(prefix) if err != nil { - return fmt.Errorf("get FrostFS object: %v", err) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) + ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) + return nil, err } - objWriter, err := h.addObjectToZip(zipWriter, &resGet.Header) + log = log.With(zap.String("cid", scid), zap.String("prefix", prefix)) + + resSearch, err := h.search(ctx, cnrID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { - return fmt.Errorf("zip create header: %v", err) + log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) + return nil, err } + return resSearch, nil +} - if _, err = io.CopyBuffer(objWriter, resGet.Payload, bufZip); err != nil { +func writeToArchive(resGet *Object, objWriter io.Writer, buf []byte) error { + var err error + if _, err = io.CopyBuffer(objWriter, resGet.Payload, buf); err != nil { return fmt.Errorf("copy object payload to zip file: %v", err) } @@ -208,14 +290,10 @@ func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid return fmt.Errorf("object body close error: %w", err) } - if err = zipWriter.Flush(); err != nil { - return fmt.Errorf("flush zip writer: %v", err) - } - return nil } -func getZipFilePath(obj *object.Object) string { +func getFilePath(obj *object.Object) string { for _, attr := range obj.Attributes() { if attr.Key() == object.AttributeFilePath { return attr.Value() diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2f1c6ad..3d2b95d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -216,7 +216,7 @@ func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path } addr := newAddress(cnrID, foundOID.OID) - handler(ctx, h.newRequest(c, log), addr) + handler(ctx, newRequest(c, log), addr) } // byAttribute is a wrapper similar to byNativeAddress. @@ -265,7 +265,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte addr.SetContainer(bktInfo.CID) addr.SetObject(objID) - handler(ctx, h.newRequest(c, log), addr) + handler(ctx, newRequest(c, log), addr) } func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { diff --git a/internal/handler/handler_fuzz_test.go b/internal/handler/handler_fuzz_test.go index ad2ae6e..d71e8b0 100644 --- a/internal/handler/handler_fuzz_test.go +++ b/internal/handler/handler_fuzz_test.go @@ -517,7 +517,7 @@ func DoFuzzDownloadZipped(input []byte) int { r.SetUserValue("cid", cid) r.SetUserValue("prefix", prefix) - hc.Handler().DownloadZipped(r) + hc.Handler().DownloadZip(r) return fuzzSuccessExitCode } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index e1bc010..4784708 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -250,7 +250,7 @@ func TestBasic(t *testing.T) { t.Run("zip", func(t *testing.T) { r = prepareGetZipped(ctx, bktName, "") - hc.Handler().DownloadZipped(r) + hc.Handler().DownloadZip(r) readerAt := bytes.NewReader(r.Response.Body()) zipReader, err := zip.NewReader(readerAt, int64(len(r.Response.Body()))) diff --git a/internal/handler/head.go b/internal/handler/head.go index da96eff..94f5ccb 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -135,7 +135,7 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { return } - req := h.newRequest(c, log) + req := newRequest(c, log) var objID oid.ID if checkS3Err == nil { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 7fdd396..971c3c8 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -24,6 +24,13 @@ type request struct { log *zap.Logger } +func newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) request { + return request{ + RequestCtx: ctx, + log: log, + } +} + func (r *request) handleFrostFSErr(err error, start time.Time) { logFields := []zap.Field{ zap.Stringer("elapsed", time.Since(start)), diff --git a/internal/logs/logs.go b/internal/logs/logs.go index f9b13b1..a4f206b 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -11,9 +11,12 @@ const ( ObjectNotFound = "object not found" // Error in ../../downloader/download.go ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go + FailedToGetObject = "failed to get object" // Error in ../../downloader/download.go IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go + CloseGzipWriter = "close gzip writer" // Error in ../../downloader/download.go + CloseTarWriter = "close tar writer" // Error in ../../downloader/download.go ServiceIsRunning = "service is running" // Info in ../../metrics/service.go ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go @@ -24,7 +27,6 @@ const ( IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go - CouldNotProcessHeaders = "could not process headers" // Error in ../../uploader/upload.go CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go From 1e7309684bb41ba096563ed8d7c89a4e0fb33148 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 6 Dec 2024 15:01:16 +0300 Subject: [PATCH 44/86] [#170] Support .tar/.tgz unpacking during upload During upload if X-Explode-Archive is set, gate tries to read archive and create an object for each file. Each object acquires a FilePath attribute which is calculated relative to the archive root. Archive could have compression via Gzip if "Content-Encoding: gzip" header is specified Signed-off-by: Nikita Zinkevich --- internal/handler/download.go | 2 +- internal/handler/multipart.go | 4 +- internal/handler/upload.go | 245 +++++++++++++++++++++++----------- internal/handler/utils.go | 7 + internal/logs/logs.go | 19 ++- 5 files changed, 185 insertions(+), 92 deletions(-) diff --git a/internal/handler/download.go b/internal/handler/download.go index 684e3b8..4641052 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -257,7 +257,7 @@ func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID } func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, cnrID cid.ID) (ResObjectSearch, error) { - scid := cnrID.EncodeToString() + scid, _ := c.UserValue("cid").(string) prefix, _ := c.UserValue("prefix").(string) ctx := utils.GetContextFromRequest(c) diff --git a/internal/handler/multipart.go b/internal/handler/multipart.go index 213286c..ebf5edd 100644 --- a/internal/handler/multipart.go +++ b/internal/handler/multipart.go @@ -42,7 +42,9 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF // ignore multipart/form-data values if filename == "" { l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name)) - + if err = part.Close(); err != nil { + l.Warn(logs.FailedToCloseReader, zap.Error(err)) + } continue } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 9493635..d1953c9 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -1,13 +1,19 @@ package handler import ( + "archive/tar" + "bytes" + "compress/gzip" "context" "encoding/json" + "errors" "io" "net/http" + "path/filepath" "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -19,8 +25,9 @@ import ( ) const ( - jsonHeader = "application/json; charset=UTF-8" - drainBufSize = 4096 + jsonHeader = "application/json; charset=UTF-8" + drainBufSize = 4096 + explodeArchiveHeader = "X-Explode-Archive" ) type putResponse struct { @@ -43,11 +50,7 @@ func (pr *putResponse) encode(w io.Writer) error { // Upload handles multipart upload request. func (h *Handler) Upload(c *fasthttp.RequestCtx) { - var ( - file MultipartFile - idObj oid.ID - addr oid.Address - ) + var file MultipartFile scid, _ := c.UserValue("cid").(string) bodyStream := c.RequestBodyStream() @@ -63,20 +66,6 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { return } - defer func() { - // If the temporary reader can be closed - let's close it. - if file == nil { - return - } - err := file.Close() - log.Debug( - logs.CloseTemporaryMultipartFormFile, - zap.Stringer("address", addr), - zap.String("filename", file.FileName()), - zap.Error(err), - ) - }() - boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) @@ -86,53 +75,69 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { filtered, err := filterHeaders(log, &c.Request.Header) if err != nil { - log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) + log.Error(logs.FailedToFilterHeaders, zap.Error(err)) ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } - now := time.Now() - if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { - if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { - log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err)) - } else { - now = parsed - } + if c.Request.Header.Peek(explodeArchiveHeader) != nil { + h.explodeArchive(request{c, log}, bktInfo, file, filtered) + } else { + h.uploadSingleObject(request{c, log}, bktInfo, file, filtered) } - if err = utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { - log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) - ResponseError(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) + // Multipart is multipart and thus can contain more than one part which + // we ignore at the moment. Also, when dealing with chunked encoding + // the last zero-length chunk might be left unread (because multipart + // reader only cares about its boundary and doesn't look further) and + // it will be (erroneously) interpreted as the start of the next + // pipelined header. Thus, we need to drain the body buffer. + for { + _, err = bodyStream.Read(drainBuf) + if err == io.EOF || errors.Is(err, io.ErrUnexpectedEOF) { + break + } + } +} + +func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file MultipartFile, filtered map[string]string) { + c, log := req.RequestCtx, req.log + setIfNotExist(filtered, object.AttributeFileName, file.FileName()) + + attributes, err := h.extractAttributes(c, log, filtered) + if err != nil { + log.Error(logs.FailedToGetAttributes, zap.Error(err)) + ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) return } - attributes := make([]object.Attribute, 0, len(filtered)) - // prepares attributes from filtered headers - for key, val := range filtered { - attribute := object.NewAttribute() - attribute.SetKey(key) - attribute.SetValue(val) - attributes = append(attributes, *attribute) + idObj, err := h.uploadObject(c, bkt, attributes, file) + if err != nil { + h.handlePutFrostFSErr(c, err, log) + return } - // sets FileName attribute if it wasn't set from header - if _, ok := filtered[object.AttributeFileName]; !ok { - filename := object.NewAttribute() - filename.SetKey(object.AttributeFileName) - filename.SetValue(file.FileName()) - attributes = append(attributes, *filename) - } - // sets Timestamp attribute if it wasn't set from header and enabled by settings - if _, ok := filtered[object.AttributeTimestamp]; !ok && h.config.DefaultTimestamp() { - timestamp := object.NewAttribute() - timestamp.SetKey(object.AttributeTimestamp) - timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) - attributes = append(attributes, *timestamp) + log.Debug(logs.ObjectUploaded, + zap.String("oid", idObj.EncodeToString()), + zap.String("FileName", file.FileName()), + ) + + addr := newAddress(bkt.CID, idObj) + c.Response.Header.SetContentType(jsonHeader) + // Try to return the response, otherwise, if something went wrong, throw an error. + if err = newPutResponse(addr).encode(c); err != nil { + log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) + ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) + return } +} + +func (h *Handler) uploadObject(c *fasthttp.RequestCtx, bkt *data.BucketInfo, attrs []object.Attribute, file io.Reader) (oid.ID, error) { + ctx := utils.GetContextFromRequest(c) obj := object.New() - obj.SetContainerID(bktInfo.CID) + obj.SetContainerID(bkt.CID) obj.SetOwnerID(*h.ownerID) - obj.SetAttributes(attributes...) + obj.SetAttributes(attrs...) prm := PrmObjectCreate{ PrmAuth: PrmAuth{ @@ -141,40 +146,120 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { Object: obj, Payload: file, ClientCut: h.config.ClientCut(), - WithoutHomomorphicHash: bktInfo.HomomorphicHashDisabled, + WithoutHomomorphicHash: bkt.HomomorphicHashDisabled, BufferMaxSize: h.config.BufferMaxSizeForPut(), } - if idObj, err = h.frostfs.CreateObject(ctx, prm); err != nil { - h.handlePutFrostFSErr(c, err, log) - return + idObj, err := h.frostfs.CreateObject(ctx, prm) + if err != nil { + return oid.ID{}, err } - addr.SetObject(idObj) - addr.SetContainer(bktInfo.CID) + return idObj, nil +} - // Try to return the response, otherwise, if something went wrong, throw an error. - if err = newPutResponse(addr).encode(c); err != nil { - log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) - ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) - - return - } - // Multipart is multipart and thus can contain more than one part which - // we ignore at the moment. Also, when dealing with chunked encoding - // the last zero-length chunk might be left unread (because multipart - // reader only cares about its boundary and doesn't look further) and - // it will be (erroneously) interpreted as the start of the next - // pipelined header. Thus we need to drain the body buffer. - for { - _, err = bodyStream.Read(drainBuf) - if err == io.EOF || err == io.ErrUnexpectedEOF { - break +func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, filtered map[string]string) ([]object.Attribute, error) { + now := time.Now() + if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { + if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { + log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err)) + } else { + now = parsed } } - // Report status code and content type. - c.Response.SetStatusCode(fasthttp.StatusOK) - c.Response.Header.SetContentType(jsonHeader) + if err := utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { + log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) + return nil, err + } + attributes := make([]object.Attribute, 0, len(filtered)) + // prepares attributes from filtered headers + for key, val := range filtered { + attribute := newAttribute(key, val) + attributes = append(attributes, attribute) + } + // sets Timestamp attribute if it wasn't set from header and enabled by settings + if _, ok := filtered[object.AttributeTimestamp]; !ok && h.config.DefaultTimestamp() { + timestamp := newAttribute(object.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10)) + attributes = append(attributes, timestamp) + } + + return attributes, nil +} + +func newAttribute(key string, val string) object.Attribute { + attr := object.NewAttribute() + attr.SetKey(key) + attr.SetValue(val) + return *attr +} + +// explodeArchive read files from archive and creates objects for each of them. +// Sets FilePath attribute with name from tar.Header. +func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.ReadCloser, filtered map[string]string) { + c, log := req.RequestCtx, req.log + + // remove user attributes which vary for each file in archive + // to guarantee that they won't appear twice + delete(filtered, object.AttributeFileName) + delete(filtered, object.AttributeFilePath) + + commonAttributes, err := h.extractAttributes(c, log, filtered) + if err != nil { + log.Error(logs.FailedToGetAttributes, zap.Error(err)) + ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) + return + } + attributes := commonAttributes + + reader := file + if bytes.EqualFold(c.Request.Header.Peek(fasthttp.HeaderContentEncoding), []byte("gzip")) { + log.Debug(logs.GzipReaderSelected) + gzipReader, err := gzip.NewReader(file) + if err != nil { + log.Error(logs.FailedToCreateGzipReader, zap.Error(err)) + ResponseError(c, "could read gzip file: "+err.Error(), fasthttp.StatusBadRequest) + return + } + defer func() { + if err := gzipReader.Close(); err != nil { + log.Warn(logs.FailedToCloseReader, zap.Error(err)) + } + }() + reader = gzipReader + } + + tarReader := tar.NewReader(reader) + for { + obj, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } else if err != nil { + log.Error(logs.FailedToReadFileFromTar, zap.Error(err)) + ResponseError(c, "could not get next entry: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + if isDir(obj.Name) { + continue + } + + // set varying attributes + attributes = attributes[:len(commonAttributes)] + fileName := filepath.Base(obj.Name) + attributes = append(attributes, newAttribute(object.AttributeFilePath, obj.Name)) + attributes = append(attributes, newAttribute(object.AttributeFileName, fileName)) + + idObj, err := h.uploadObject(c, bkt, attributes, tarReader) + if err != nil { + h.handlePutFrostFSErr(c, err, log) + return + } + + log.Debug(logs.ObjectUploaded, + zap.String("oid", idObj.EncodeToString()), + zap.String("FileName", fileName), + ) + } } func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *zap.Logger) { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 971c3c8..74932f3 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -101,6 +101,13 @@ func newAddress(cnr cid.ID, obj oid.ID) oid.Address { return addr } +// setIfNotExist sets key value to map if key is not present yet. +func setIfNotExist(m map[string]string, key, value string) { + if _, ok := m[key]; !ok { + m[key] = value + } +} + func ResponseError(r *fasthttp.RequestCtx, msg string, code int) { r.Error(msg+"\n", code) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a4f206b..68270ed 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -4,8 +4,6 @@ const ( CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/* CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go - WrongObjectID = "wrong object id" // Error in ../../downloader/download.go - GetLatestObjectVersion = "get latest object version" // Error in ../../downloader/download.go ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go ObjectNotFound = "object not found" // Error in ../../downloader/download.go @@ -15,8 +13,6 @@ const ( IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go - CloseGzipWriter = "close gzip writer" // Error in ../../downloader/download.go - CloseTarWriter = "close tar writer" // Error in ../../downloader/download.go ServiceIsRunning = "service is running" // Info in ../../metrics/service.go ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go @@ -25,7 +21,6 @@ const ( CantGracefullyShutDownService = "can't gracefully shut down service, force stop" // Error in ../../metrics/service.go IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go - CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go @@ -81,11 +76,6 @@ const ( InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go FailedToUnescapeQuery = "failed to unescape query" - FailedToParseAddressInTreeNode = "failed to parse object addr in tree node" - SettingsNodeInvalidOwnerKey = "settings node: invalid owner key" - SystemNodeHasMultipleIDs = "system node has multiple ids" - FailedToRemoveOldSystemNode = "failed to remove old system node" - BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" @@ -96,4 +86,13 @@ const ( MultinetConfigWontBeUpdated = "multinet config won't be updated" ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" CouldntCacheNetmap = "couldn't cache netmap" + FailedToFilterHeaders = "failed to filter headers" + FailedToReadFileFromTar = "failed to read file from tar" + FailedToGetAttributes = "failed to get attributes" + ObjectUploaded = "object uploaded" + CloseGzipWriter = "close gzip writer" + CloseTarWriter = "close tar writer" + FailedToCloseReader = "failed to close reader" + FailedToCreateGzipReader = "failed to create gzip reader" + GzipReaderSelected = "gzip reader selected" ) From 1e897aa3c307df14bdbec1d5b690e8c187a88544 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 6 Dec 2024 15:01:16 +0300 Subject: [PATCH 45/86] [#170] Updated docs and configuration of archive section Signed-off-by: Nikita Zinkevich --- docs/api.md | 37 +++++++++++++++++--------------- internal/handler/download.go | 4 ++-- internal/handler/handler.go | 2 +- internal/handler/handler_test.go | 2 +- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/docs/api.md b/docs/api.md index e59956a..d099915 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,11 +1,11 @@ # HTTP Gateway Specification -| Route | Description | -|-------------------------------------------------|----------------------------------------------| -| `/upload/{cid}` | [Put object](#put-object) | -| `/get/{cid}/{oid}` | [Get object](#get-object) | -| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) | -| `/zip/{cid}/{prefix}` | [Download objects in archive](#download-zip) | +| Route | Description | +|-------------------------------------------------|--------------------------------------------------| +| `/upload/{cid}` | [Put object](#put-object) | +| `/get/{cid}/{oid}` | [Get object](#get-object) | +| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) | +| `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}` | [Download objects in archive](#download-archive) | **Note:** `cid` parameter can be base58 encoded container ID or container name (the name must be registered in NNS, see appropriate section in [nns.md](./nns.md)). @@ -56,12 +56,14 @@ Upload file as object with attributes to FrostFS. ###### Headers -| Header | Description | -|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| -| Common headers | See [bearer token](#bearer-token). | -| `X-Attribute-System-*` | Used to set system FrostFS object attributes
(e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). | -| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | -| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | +| Header | Description | +|------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Common headers | See [bearer token](#bearer-token). | +| `X-Attribute-System-*` | Used to set system FrostFS object attributes
(e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). | +| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | +| `X-Explode-Archive` | If set, gate tries to read files from uploading `tar` archive and creates an object for each file in it. Uploading `tar` could be compressed via Gzip by setting a `Content-Encoding` header. Sets a `FilePath` attribute as a relative path from archive root and a `FileName` as the last path element of the `FilePath`. | +| `Content-Encoding` | If set and value is `gzip`, gate will handle uploading file as a `Gzip` compressed `tar` file. | +| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority): @@ -269,9 +271,9 @@ If more than one object is found, an arbitrary one will be used to get attribute | 400 | Some error occurred during operation. | | 404 | Container or object not found. | -## Download zip +## Download archive -Route: `/zip/{cid}/{prefix}` +Route: `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}` | Route parameter | Type | Description | |-----------------|-----------|---------------------------------------------------------| @@ -282,12 +284,13 @@ Route: `/zip/{cid}/{prefix}` #### GET -Find objects by prefix for `FilePath` attributes. Return found objects in zip archive. +Find objects by prefix for `FilePath` attributes. Return found objects in zip or tar archive. Name of files in archive sets to `FilePath` attribute of objects. Time of files sets to time when object has started downloading. -You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` route. +You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` or +`/tar/{cid}/` route. -Archive can be compressed (see http-gw [configuration](gate-configuration.md#zip-section)). +Archive can be compressed (see http-gw [configuration](gate-configuration.md#archive-section)). ##### Request diff --git a/internal/handler/download.go b/internal/handler/download.go index 4641052..d5fac23 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -134,7 +134,7 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, error) { method := zip.Store - if h.config.ZipCompression() { + if h.config.ArchiveCompression() { method = zip.Deflate } @@ -177,7 +177,7 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res defer resSearch.Close() compressionLevel := gzip.NoCompression - if h.config.ZipCompression() { + if h.config.ArchiveCompression() { compressionLevel = gzip.DefaultCompression } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 3d2b95d..4e23c3b 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -28,7 +28,7 @@ import ( type Config interface { DefaultTimestamp() bool - ZipCompression() bool + ArchiveCompression() bool ClientCut() bool IndexPageEnabled() bool IndexPageTemplate() string diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 4784708..53c9739 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -66,7 +66,7 @@ func (c *configMock) DefaultTimestamp() bool { return false } -func (c *configMock) ZipCompression() bool { +func (c *configMock) ArchiveCompression() bool { return false } From 36bd3e2d43e1615fc7fed32e8874fd9db5eb8562 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Wed, 25 Dec 2024 13:08:22 +0300 Subject: [PATCH 46/86] [#170] logs: Remove comments Signed-off-by: Nikita Zinkevich --- internal/logs/logs.go | 148 +++++++++++++++++++++--------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 68270ed..5f60b9b 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -1,80 +1,80 @@ package logs const ( - CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/* - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go - CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go - ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go - CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go - ObjectNotFound = "object not found" // Error in ../../downloader/download.go - ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go - FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go - FailedToGetObject = "failed to get object" // Error in ../../downloader/download.go - IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go - ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go - CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go - ServiceIsRunning = "service is running" // Info in ../../metrics/service.go - ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go - ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go - ShuttingDownService = "shutting down service" // Info in ../../metrics/service.go - CantShutDownService = "can't shut down service" // Panic in ../../metrics/service.go - CantGracefullyShutDownService = "can't gracefully shut down service, force stop" // Error in ../../metrics/service.go - IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go - IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go - CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go - CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go - CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go - CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go - CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go - AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go - FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go - FailedToCreateWorkerPool = "failed to create worker pool" // Fatal in ../../app.go - FailedToReadIndexPageTemplate = "failed to read index page template" // Error in ../../app.go - SetCustomIndexPageTemplate = "set custom index page template" // Info in ../../app.go - ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go - MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go - NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go - StartingApplication = "starting application" // Info in ../../app.go - StartingServer = "starting server" // Info in ../../app.go - ListenAndServe = "listen and serve" // Fatal in ../../app.go - ShuttingDownWebServer = "shutting down web server" // Info in ../../app.go - FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../app.go - SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../app.go - FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../app.go - FailedToReloadConfig = "failed to reload config" // Warn in ../../app.go - LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../app.go - FailedToUpdateResolvers = "failed to update resolvers" // Warn in ../../app.go - FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../app.go - SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../app.go - AddedPathUploadCid = "added path /upload/{cid}" // Info in ../../app.go - AddedPathGetCidOid = "added path /get/{cid}/{oid}" // Info in ../../app.go - AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" // Info in ../../app.go - AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" // Info in ../../app.go - Request = "request" // Info in ../../app.go - CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" // Error in ../../app.go - FailedToAddServer = "failed to add server" // Warn in ../../app.go - AddServer = "add server" // Info in ../../app.go - NoHealthyServers = "no healthy servers" // Fatal in ../../app.go - FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../app.go - TracingConfigUpdated = "tracing config updated" // Info in ../../app.go - ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" // Warn in ../../app.go - RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" // Warn in ../../app.go - RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" // Info in ../../app.go - CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../settings.go - UsingCredentials = "using credentials" // Info in ../../settings.go - FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../settings.go - FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../settings.go - FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go - FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go - AddedStoragePeer = "added storage peer" // Info in ../../settings.go - CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go - CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go - FailedToSumbitTaskToPool = "failed to submit task to pool" // Error in ../handler/browse.go - FailedToHeadObject = "failed to head object" // Error in ../handler/browse.go - FailedToIterateOverResponse = "failed to iterate over search response" // Error in ../handler/browse.go - InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go - InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go - InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go + CouldntParseCreationDate = "couldn't parse creation date" + CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" + CouldNotReceiveObject = "could not receive object" + ObjectWasDeleted = "object was deleted" + CouldNotSearchForObjects = "could not search for objects" + ObjectNotFound = "object not found" + ReadObjectListFailed = "read object list failed" + FailedToAddObjectToArchive = "failed to add object to archive" + FailedToGetObject = "failed to get object" + IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" + ObjectsNotFound = "objects not found" + CloseZipWriter = "close zip writer" + ServiceIsRunning = "service is running" + ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" + ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" + ShuttingDownService = "shutting down service" + CantShutDownService = "can't shut down service" + CantGracefullyShutDownService = "can't gracefully shut down service, force stop" + IgnorePartEmptyFormName = "ignore part, empty form name" + IgnorePartEmptyFilename = "ignore part, empty filename" + CouldNotReceiveMultipartForm = "could not receive multipart/form" + CouldNotParseClientTime = "could not parse client time" + CouldNotPrepareExpirationHeader = "could not prepare expiration header" + CouldNotEncodeResponse = "could not encode response" + CouldNotStoreFileInFrostfs = "could not store file in frostfs" + AddAttributeToResultObject = "add attribute to result object" + FailedToCreateResolver = "failed to create resolver" + FailedToCreateWorkerPool = "failed to create worker pool" + FailedToReadIndexPageTemplate = "failed to read index page template" + SetCustomIndexPageTemplate = "set custom index page template" + ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" + MetricsAreDisabled = "metrics are disabled" + NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" + StartingApplication = "starting application" + StartingServer = "starting server" + ListenAndServe = "listen and serve" + ShuttingDownWebServer = "shutting down web server" + FailedToShutdownTracing = "failed to shutdown tracing" + SIGHUPConfigReloadStarted = "SIGHUP config reload started" + FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" + FailedToReloadConfig = "failed to reload config" + LogLevelWontBeUpdated = "log level won't be updated" + FailedToUpdateResolvers = "failed to update resolvers" + FailedToReloadServerParameters = "failed to reload server parameters" + SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" + AddedPathUploadCid = "added path /upload/{cid}" + AddedPathGetCidOid = "added path /get/{cid}/{oid}" + AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" + AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" + Request = "request" + CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" + FailedToAddServer = "failed to add server" + AddServer = "add server" + NoHealthyServers = "no healthy servers" + FailedToInitializeTracing = "failed to initialize tracing" + TracingConfigUpdated = "tracing config updated" + ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" + RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" + RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" + CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" + UsingCredentials = "using credentials" + FailedToCreateConnectionPool = "failed to create connection pool" + FailedToDialConnectionPool = "failed to dial connection pool" + FailedToCreateTreePool = "failed to create tree pool" + FailedToDialTreePool = "failed to dial tree pool" + AddedStoragePeer = "added storage peer" + CouldntGetBucket = "could not get bucket" + CouldntPutBucketIntoCache = "couldn't put bucket info into cache" + FailedToSumbitTaskToPool = "failed to submit task to pool" + FailedToHeadObject = "failed to head object" + FailedToIterateOverResponse = "failed to iterate over search response" + InvalidCacheEntryType = "invalid cache entry type" + InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" + InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" FailedToUnescapeQuery = "failed to unescape query" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" From 87ace4f8f7eee2cc128aa274fa0b19be55e441c5 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 28 Jan 2025 17:43:11 +0300 Subject: [PATCH 47/86] [#201] govulncheck: Use patch release with security fixes https://go.dev/doc/devel/release#go1.23.minor Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/vulncheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 76e2965..529e8a8 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -16,7 +16,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.22.11' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest From 526da379ad64439ed5e4e92774c8c9e57ff81564 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 27 Jan 2025 12:46:26 +0300 Subject: [PATCH 48/86] [#199] Fix SIGHUP panic Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 56c6be1..35a46ba 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -130,6 +130,7 @@ func newApp(ctx context.Context, v *viper.Viper) App { a := &app{ ctx: ctx, log: log.logger, + logLevel: log.lvl, cfg: v, loggerSettings: logSettings, webServer: new(fasthttp.Server), From a6fdaf9456bad2799dcf36ee95731e45c32505c8 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 27 Jan 2025 13:16:48 +0300 Subject: [PATCH 49/86] [#199] Clear app services list Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 35a46ba..3df637b 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -616,6 +616,8 @@ func (a *app) configReload(ctx context.Context) { } func (a *app) startServices() { + a.services = a.services[:0] + pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)} pprofService := metrics.NewPprofService(a.log, pprofConfig) a.services = append(a.services, pprofService) From 8de06e23a07552f0083a4f6bac54dea1bce485bd Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 27 Jan 2025 13:17:02 +0300 Subject: [PATCH 50/86] [#199] Use default value if config param is unset after SIGHUP Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 72 ++++----- cmd/http-gw/integration_test.go | 17 +-- cmd/http-gw/main.go | 4 +- cmd/http-gw/settings.go | 253 ++++++++++++++++++++------------ cmd/http-gw/settings_test.go | 60 ++++++++ 5 files changed, 269 insertions(+), 137 deletions(-) create mode 100644 cmd/http-gw/settings_test.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 3df637b..ac20d29 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -56,7 +56,7 @@ type ( treePool *treepool.Pool key *keys.PrivateKey owner *user.ID - cfg *viper.Viper + cfg *appCfg webServer *fasthttp.Server webDone chan struct{} resolver *resolver.ContainerResolver @@ -123,35 +123,35 @@ type ( } ) -func newApp(ctx context.Context, v *viper.Viper) App { +func newApp(ctx context.Context, cfg *appCfg) App { logSettings := &loggerSettings{} - log := pickLogger(v, logSettings) + log := pickLogger(cfg.config(), logSettings) a := &app{ ctx: ctx, log: log.logger, logLevel: log.lvl, - cfg: v, + cfg: cfg, loggerSettings: logSettings, webServer: new(fasthttp.Server), webDone: make(chan struct{}), - bucketCache: cache.NewBucketCache(getBucketCacheOptions(v, log.logger), v.GetBool(cfgFeaturesTreePoolNetmapSupport)), + bucketCache: cache.NewBucketCache(getBucketCacheOptions(cfg.config(), log.logger), cfg.config().GetBool(cfgFeaturesTreePoolNetmapSupport)), } a.initAppSettings() // -- setup FastHTTP server -- a.webServer.Name = "frost-http-gw" - a.webServer.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) - a.webServer.WriteBufferSize = a.cfg.GetInt(cfgWebWriteBufferSize) - a.webServer.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout) - a.webServer.WriteTimeout = a.cfg.GetDuration(cfgWebWriteTimeout) + a.webServer.ReadBufferSize = a.config().GetInt(cfgWebReadBufferSize) + a.webServer.WriteBufferSize = a.config().GetInt(cfgWebWriteBufferSize) + a.webServer.ReadTimeout = a.config().GetDuration(cfgWebReadTimeout) + a.webServer.WriteTimeout = a.config().GetDuration(cfgWebWriteTimeout) a.webServer.DisableHeaderNamesNormalizing = true a.webServer.NoDefaultServerHeader = true a.webServer.NoDefaultContentType = true - a.webServer.MaxRequestBodySize = a.cfg.GetInt(cfgWebMaxRequestBodySize) + a.webServer.MaxRequestBodySize = a.config().GetInt(cfgWebMaxRequestBodySize) a.webServer.DisablePreParseMultipartForm = true - a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) + a.webServer.StreamRequestBody = a.config().GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- a.initPools(ctx) @@ -168,13 +168,17 @@ func newApp(ctx context.Context, v *viper.Viper) App { return a } +func (a *app) config() *viper.Viper { + return a.cfg.config() +} + func (a *app) initAppSettings() { a.settings = &appSettings{ - reconnectInterval: fetchReconnectInterval(a.cfg), - dialerSource: getDialerSource(a.log, a.cfg), - workerPoolSize: a.cfg.GetInt(cfgWorkerPoolSize), + reconnectInterval: fetchReconnectInterval(a.config()), + dialerSource: getDialerSource(a.log, a.config()), + workerPoolSize: a.config().GetInt(cfgWorkerPoolSize), } - a.settings.update(a.cfg, a.log) + a.settings.update(a.config(), a.log) } func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { @@ -327,11 +331,11 @@ func (a *app) initResolver() { func (a *app) getResolverConfig() ([]string, *resolver.Config) { resolveCfg := &resolver.Config{ FrostFS: frostfs.NewResolverFrostFS(a.pool), - RPCAddress: a.cfg.GetString(cfgRPCEndpoint), + RPCAddress: a.config().GetString(cfgRPCEndpoint), Settings: a.settings, } - order := a.cfg.GetStringSlice(cfgResolveOrder) + order := a.config().GetStringSlice(cfgResolveOrder) if resolveCfg.RPCAddress == "" { order = remove(order, resolver.NNSResolver) a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided) @@ -346,7 +350,7 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) { func (a *app) initMetrics() { gateMetricsProvider := metrics.NewGateMetrics(a.pool) - a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled)) + a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.config().GetBool(cfgPrometheusEnabled)) a.metrics.SetHealth(metrics.HealthStatusStarting) a.loggerSettings.setMetrics(a.metrics.provider) } @@ -574,22 +578,22 @@ func (a *app) shutdownTracing() { func (a *app) configReload(ctx context.Context) { a.log.Info(logs.SIGHUPConfigReloadStarted) - if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) { + if !a.config().IsSet(cmdConfig) && !a.config().IsSet(cmdConfigDir) { a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed) return } - if err := readInConfig(a.cfg); err != nil { + if err := a.cfg.reload(); err != nil { a.log.Warn(logs.FailedToReloadConfig, zap.Error(err)) return } - if lvl, err := getLogLevel(a.cfg); err != nil { + if lvl, err := getLogLevel(a.config()); err != nil { a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err)) } else { a.logLevel.SetLevel(lvl) } - if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.cfg, a.log)); err != nil { + if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil { a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err)) } @@ -606,9 +610,9 @@ func (a *app) configReload(ctx context.Context) { a.stopServices() a.startServices() - a.settings.update(a.cfg, a.log) + a.settings.update(a.config(), a.log) - a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) + a.metrics.SetEnabled(a.config().GetBool(cfgPrometheusEnabled)) a.initTracing(ctx) a.setHealthStatus() @@ -618,12 +622,12 @@ func (a *app) configReload(ctx context.Context) { func (a *app) startServices() { a.services = a.services[:0] - pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)} + pprofConfig := metrics.Config{Enabled: a.config().GetBool(cfgPprofEnabled), Address: a.config().GetString(cfgPprofAddress)} pprofService := metrics.NewPprofService(a.log, pprofConfig) a.services = append(a.services, pprofService) go pprofService.Start() - prometheusConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPrometheusEnabled), Address: a.cfg.GetString(cfgPrometheusAddress)} + prometheusConfig := metrics.Config{Enabled: a.config().GetBool(cfgPrometheusEnabled), Address: a.config().GetString(cfgPrometheusAddress)} prometheusService := metrics.NewPrometheusService(a.log, prometheusConfig) a.services = append(a.services, prometheusService) go prometheusService.Start() @@ -850,7 +854,7 @@ func (a *app) AppParams() *handler.AppParams { } func (a *app) initServers(ctx context.Context) { - serversInfo := fetchServers(a.cfg, a.log) + serversInfo := fetchServers(a.config(), a.log) a.servers = make([]Server, 0, len(serversInfo)) for _, serverInfo := range serversInfo { @@ -877,7 +881,7 @@ func (a *app) initServers(ctx context.Context) { } func (a *app) updateServers() error { - serversInfo := fetchServers(a.cfg, a.log) + serversInfo := fetchServers(a.config(), a.log) a.mu.Lock() defer a.mu.Unlock() @@ -935,15 +939,15 @@ func (a *app) initTracing(ctx context.Context) { instanceID = a.servers[0].Address() } cfg := tracing.Config{ - Enabled: a.cfg.GetBool(cfgTracingEnabled), - Exporter: tracing.Exporter(a.cfg.GetString(cfgTracingExporter)), - Endpoint: a.cfg.GetString(cfgTracingEndpoint), + Enabled: a.config().GetBool(cfgTracingEnabled), + Exporter: tracing.Exporter(a.config().GetString(cfgTracingExporter)), + Endpoint: a.config().GetString(cfgTracingEndpoint), Service: "frostfs-http-gw", InstanceID: instanceID, Version: Version, } - if trustedCa := a.cfg.GetString(cfgTracingTrustedCa); trustedCa != "" { + if trustedCa := a.config().GetString(cfgTracingTrustedCa); trustedCa != "" { caBytes, err := os.ReadFile(trustedCa) if err != nil { a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) @@ -958,7 +962,7 @@ func (a *app) initTracing(ctx context.Context) { cfg.ServerCaCertPool = certPool } - attributes, err := fetchTracingAttributes(a.cfg) + attributes, err := fetchTracingAttributes(a.config()) if err != nil { a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) return @@ -981,7 +985,7 @@ func (a *app) setRuntimeParameters() { return } - softMemoryLimit := fetchSoftMemoryLimit(a.cfg) + softMemoryLimit := fetchSoftMemoryLimit(a.config()) previous := debug.SetMemoryLimit(softMemoryLimit) if softMemoryLimit != previous { a.log.Info(logs.RuntimeSoftMemoryLimitUpdated, diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index c3c5de5..2596bee 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -32,7 +32,6 @@ import ( docker "github.com/docker/docker/api/types/container" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/wallet" - "github.com/spf13/viper" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" @@ -108,8 +107,8 @@ func runServer(pathToWallet string) (App, context.CancelFunc) { cancelCtx, cancel := context.WithCancel(context.Background()) v := getDefaultConfig() - v.Set(cfgWalletPath, pathToWallet) - v.Set(cfgWalletPassphrase, "") + v.config().Set(cfgWalletPath, pathToWallet) + v.config().Set(cfgWalletPassphrase, "") application := newApp(cancelCtx, v) go application.Serve() @@ -452,14 +451,14 @@ func createDockerContainer(ctx context.Context, t *testing.T, image string) test return aioC } -func getDefaultConfig() *viper.Viper { +func getDefaultConfig() *appCfg { v := settings() - v.SetDefault(cfgPeers+".0.address", "localhost:8080") - v.SetDefault(cfgPeers+".0.weight", 1) - v.SetDefault(cfgPeers+".0.priority", 1) + v.config().SetDefault(cfgPeers+".0.address", "localhost:8080") + v.config().SetDefault(cfgPeers+".0.weight", 1) + v.config().SetDefault(cfgPeers+".0.priority", 1) - v.SetDefault(cfgRPCEndpoint, "http://localhost:30333") - v.SetDefault("server.0.address", testListenAddress) + v.config().SetDefault(cfgRPCEndpoint, "http://localhost:30333") + v.config().SetDefault("server.0.address", testListenAddress) return v } diff --git a/cmd/http-gw/main.go b/cmd/http-gw/main.go index fdd148c..002f190 100644 --- a/cmd/http-gw/main.go +++ b/cmd/http-gw/main.go @@ -8,9 +8,9 @@ import ( func main() { globalContext, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - v := settings() + cfg := settings() - application := newApp(globalContext, v) + application := newApp(globalContext, cfg) go application.Serve() application.Wait() } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 5cf06a0..5670f87 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -12,6 +12,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" @@ -197,14 +198,72 @@ type Logger struct { lvl zap.AtomicLevel } -func settings() *viper.Viper { +type appCfg struct { + flags *pflag.FlagSet + + mu sync.RWMutex + settings *viper.Viper +} + +func (a *appCfg) reload() error { + old := a.config() + + v, err := newViper(a.flags) + if err != nil { + return err + } + + if old.IsSet(cmdConfig) { + v.Set(cmdConfig, old.Get(cmdConfig)) + } + if old.IsSet(cmdConfigDir) { + v.Set(cmdConfigDir, old.Get(cmdConfigDir)) + } + + if err = readInConfig(v); err != nil { + return err + } + + a.setConfig(v) + return nil +} + +func (a *appCfg) config() *viper.Viper { + a.mu.RLock() + defer a.mu.RUnlock() + + return a.settings +} + +func (a *appCfg) setConfig(v *viper.Viper) { + a.mu.Lock() + a.settings = v + a.mu.Unlock() +} + +func newViper(flags *pflag.FlagSet) (*viper.Viper, error) { v := viper.New() + v.AutomaticEnv() v.SetEnvPrefix(Prefix) v.AllowEmptyEnv(true) v.SetConfigType("yaml") v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + if err := bindFlags(v, flags); err != nil { + return nil, err + } + + setDefaults(v, flags) + + if v.IsSet(cfgServer+".0."+cfgTLSKeyFile) && v.IsSet(cfgServer+".0."+cfgTLSCertFile) { + v.Set(cfgServer+".0."+cfgTLSEnabled, true) + } + + return v, nil +} + +func settings() *appCfg { // flags setup: flags := pflag.NewFlagSet("commandline", pflag.ExitOnError) flags.SetOutput(os.Stdout) @@ -228,89 +287,17 @@ func settings() *viper.Viper { flags.String(cmdListenAddress, "0.0.0.0:8080", "addresses to listen") flags.String(cfgTLSCertFile, "", "TLS certificate path") flags.String(cfgTLSKeyFile, "", "TLS key path") - peers := flags.StringArrayP(cfgPeers, "p", nil, "FrostFS nodes") + flags.StringArrayP(cfgPeers, "p", nil, "FrostFS nodes") - resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order") - - // set defaults: - - // logger: - v.SetDefault(cfgLoggerLevel, "debug") - v.SetDefault(cfgLoggerDestination, "stdout") - v.SetDefault(cfgLoggerSamplingEnabled, false) - v.SetDefault(cfgLoggerSamplingThereafter, 100) - v.SetDefault(cfgLoggerSamplingInitial, 100) - v.SetDefault(cfgLoggerSamplingInterval, defaultLoggerSamplerInterval) - - // pool: - v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) - - // frostfs: - v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) - - // web-server: - v.SetDefault(cfgWebReadBufferSize, 4096) - v.SetDefault(cfgWebWriteBufferSize, 4096) - v.SetDefault(cfgWebReadTimeout, time.Minute*10) - v.SetDefault(cfgWebWriteTimeout, time.Minute*5) - v.SetDefault(cfgWebStreamRequestBody, true) - v.SetDefault(cfgWebMaxRequestBodySize, fasthttp.DefaultMaxRequestBodySize) - - v.SetDefault(cfgWorkerPoolSize, 1000) - // upload header - v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) - - // metrics - v.SetDefault(cfgPprofAddress, "localhost:8083") - v.SetDefault(cfgPrometheusAddress, "localhost:8084") - - // resolve bucket - v.SetDefault(cfgResolveNamespaceHeader, defaultNamespaceHeader) - v.SetDefault(cfgResolveDefaultNamespaces, []string{"", "root"}) - - // multinet - v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay) - - // Binding flags - if err := v.BindPFlag(cfgPprofEnabled, flags.Lookup(cmdPprof)); err != nil { - panic(err) - } - if err := v.BindPFlag(cfgPrometheusEnabled, flags.Lookup(cmdMetrics)); err != nil { - panic(err) - } - - if err := v.BindPFlag(cfgWalletPath, flags.Lookup(cmdWallet)); err != nil { - panic(err) - } - - if err := v.BindPFlag(cfgWalletAddress, flags.Lookup(cmdAddress)); err != nil { - panic(err) - } - - if err := v.BindPFlags(flags); err != nil { - panic(err) - } - - if err := v.BindPFlag(cfgServer+".0.address", flags.Lookup(cmdListenAddress)); err != nil { - panic(err) - } - if err := v.BindPFlag(cfgServer+".0."+cfgTLSKeyFile, flags.Lookup(cfgTLSKeyFile)); err != nil { - panic(err) - } - if err := v.BindPFlag(cfgServer+".0."+cfgTLSCertFile, flags.Lookup(cfgTLSCertFile)); err != nil { - panic(err) - } + flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order") if err := flags.Parse(os.Args); err != nil { panic(err) } - if v.IsSet(cfgServer+".0."+cfgTLSKeyFile) && v.IsSet(cfgServer+".0."+cfgTLSCertFile) { - v.Set(cfgServer+".0."+cfgTLSEnabled, true) - } - - if resolveMethods != nil { - v.SetDefault(cfgResolveOrder, *resolveMethods) + v, err := newViper(flags) + if err != nil { + panic(fmt.Errorf("bind flags: %w", err)) } switch { @@ -355,15 +342,97 @@ func settings() *viper.Viper { panic(err) } - if peers != nil && len(*peers) > 0 { - for i := range *peers { - v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", (*peers)[i]) + return &appCfg{ + flags: flags, + settings: v, + } +} + +func setDefaults(v *viper.Viper, flags *pflag.FlagSet) { + // set defaults: + + // logger: + v.SetDefault(cfgLoggerLevel, "debug") + v.SetDefault(cfgLoggerDestination, "stdout") + v.SetDefault(cfgLoggerSamplingEnabled, false) + v.SetDefault(cfgLoggerSamplingThereafter, 100) + v.SetDefault(cfgLoggerSamplingInitial, 100) + v.SetDefault(cfgLoggerSamplingInterval, defaultLoggerSamplerInterval) + + // pool: + v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) + + // frostfs: + v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) + + // web-server: + v.SetDefault(cfgWebReadBufferSize, 4096) + v.SetDefault(cfgWebWriteBufferSize, 4096) + v.SetDefault(cfgWebReadTimeout, time.Minute*10) + v.SetDefault(cfgWebWriteTimeout, time.Minute*5) + v.SetDefault(cfgWebStreamRequestBody, true) + v.SetDefault(cfgWebMaxRequestBodySize, fasthttp.DefaultMaxRequestBodySize) + + v.SetDefault(cfgWorkerPoolSize, 1000) + // upload header + v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) + + // metrics + v.SetDefault(cfgPprofAddress, "localhost:8083") + v.SetDefault(cfgPrometheusAddress, "localhost:8084") + + // resolve bucket + v.SetDefault(cfgResolveNamespaceHeader, defaultNamespaceHeader) + v.SetDefault(cfgResolveDefaultNamespaces, []string{"", "root"}) + + // multinet + v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay) + + if resolveMethods, err := flags.GetStringSlice(cfgResolveOrder); err == nil { + v.SetDefault(cfgResolveOrder, resolveMethods) + } + + if peers, err := flags.GetStringArray(cfgPeers); err == nil { + for i := range peers { + v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", peers[i]) v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".weight", 1) v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".priority", 1) } } +} - return v +func bindFlags(v *viper.Viper, flags *pflag.FlagSet) error { + // Binding flags + if err := v.BindPFlag(cfgPprofEnabled, flags.Lookup(cmdPprof)); err != nil { + return err + } + if err := v.BindPFlag(cfgPrometheusEnabled, flags.Lookup(cmdMetrics)); err != nil { + return err + } + + if err := v.BindPFlag(cfgWalletPath, flags.Lookup(cmdWallet)); err != nil { + return err + } + + if err := v.BindPFlag(cfgWalletAddress, flags.Lookup(cmdAddress)); err != nil { + return err + } + + if err := v.BindPFlags(flags); err != nil { + return err + } + + if err := v.BindPFlag(cfgServer+".0.address", flags.Lookup(cmdListenAddress)); err != nil { + return err + } + if err := v.BindPFlag(cfgServer+".0."+cfgTLSKeyFile, flags.Lookup(cfgTLSKeyFile)); err != nil { + return err + } + if err := v.BindPFlag(cfgServer+".0."+cfgTLSCertFile, flags.Lookup(cfgTLSCertFile)); err != nil { + return err + } + + return nil } func readInConfig(v *viper.Viper) error { @@ -616,7 +685,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { } func (a *app) initPools(ctx context.Context) { - key, err := getFrostFSKey(a.cfg, a.log) + key, err := getFrostFSKey(a.config(), a.log) if err != nil { a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) } @@ -628,40 +697,40 @@ func (a *app) initPools(ctx context.Context) { prmTree.SetKey(key) a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) - for _, peer := range fetchPeers(a.log, a.cfg) { + for _, peer := range fetchPeers(a.log, a.config()) { prm.AddNode(peer) prmTree.AddNode(peer) } - connTimeout := a.cfg.GetDuration(cfgConTimeout) + connTimeout := a.config().GetDuration(cfgConTimeout) if connTimeout <= 0 { connTimeout = defaultConnectTimeout } prm.SetNodeDialTimeout(connTimeout) prmTree.SetNodeDialTimeout(connTimeout) - streamTimeout := a.cfg.GetDuration(cfgStreamTimeout) + streamTimeout := a.config().GetDuration(cfgStreamTimeout) if streamTimeout <= 0 { streamTimeout = defaultStreamTimeout } prm.SetNodeStreamTimeout(streamTimeout) prmTree.SetNodeStreamTimeout(streamTimeout) - healthCheckTimeout := a.cfg.GetDuration(cfgReqTimeout) + healthCheckTimeout := a.config().GetDuration(cfgReqTimeout) if healthCheckTimeout <= 0 { healthCheckTimeout = defaultRequestTimeout } prm.SetHealthcheckTimeout(healthCheckTimeout) prmTree.SetHealthcheckTimeout(healthCheckTimeout) - rebalanceInterval := a.cfg.GetDuration(cfgRebalance) + rebalanceInterval := a.config().GetDuration(cfgRebalance) if rebalanceInterval <= 0 { rebalanceInterval = defaultRebalanceTimer } prm.SetClientRebalanceInterval(rebalanceInterval) prmTree.SetClientRebalanceInterval(rebalanceInterval) - errorThreshold := a.cfg.GetUint32(cfgPoolErrorThreshold) + errorThreshold := a.config().GetUint32(cfgPoolErrorThreshold) if errorThreshold <= 0 { errorThreshold = defaultPoolErrorThreshold } @@ -669,7 +738,7 @@ func (a *app) initPools(ctx context.Context) { prm.SetLogger(a.log) prmTree.SetLogger(a.log) - prmTree.SetMaxRequestAttempts(a.cfg.GetInt(cfgTreePoolMaxAttempts)) + prmTree.SetMaxRequestAttempts(a.config().GetInt(cfgTreePoolMaxAttempts)) interceptors := []grpc.DialOption{ grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), @@ -688,8 +757,8 @@ func (a *app) initPools(ctx context.Context) { a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) } - if a.cfg.GetBool(cfgFeaturesTreePoolNetmapSupport) { - prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p), cache.NewNetmapCache(getNetmapCacheOptions(a.cfg, a.log)), a.bucketCache, a.log)) + if a.config().GetBool(cfgFeaturesTreePoolNetmapSupport) { + prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p), cache.NewNetmapCache(getNetmapCacheOptions(a.config(), a.log)), a.bucketCache, a.log)) } treePool, err := treepool.NewPool(prmTree) diff --git a/cmd/http-gw/settings_test.go b/cmd/http-gw/settings_test.go new file mode 100644 index 0000000..13bd50d --- /dev/null +++ b/cmd/http-gw/settings_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "os" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "github.com/stretchr/testify/require" +) + +func TestConfigReload(t *testing.T) { + f, err := os.CreateTemp("", "conf") + require.NoError(t, err) + defer func() { + require.NoError(t, os.Remove(f.Name())) + }() + + confData := ` +pprof: + enabled: true + +resolve_bucket: + default_namespaces: [""] + +resolve_order: + - nns +` + + _, err = f.WriteString(confData) + require.NoError(t, err) + require.NoError(t, f.Close()) + + cfg := settings() + + require.NoError(t, cfg.flags.Parse([]string{"--config", f.Name(), "--connect_timeout", "15s"})) + require.NoError(t, cfg.reload()) + + require.True(t, cfg.config().GetBool(cfgPprofEnabled)) + require.Equal(t, []string{""}, cfg.config().GetStringSlice(cfgResolveDefaultNamespaces)) + require.Equal(t, []string{resolver.NNSResolver}, cfg.config().GetStringSlice(cfgResolveOrder)) + require.Equal(t, 15*time.Second, cfg.config().GetDuration(cfgConTimeout)) + + require.NoError(t, os.Truncate(f.Name(), 0)) + require.NoError(t, cfg.reload()) + + require.False(t, cfg.config().GetBool(cfgPprofEnabled)) + require.Equal(t, []string{"", "root"}, cfg.config().GetStringSlice(cfgResolveDefaultNamespaces)) + require.Equal(t, []string{resolver.NNSResolver, resolver.DNSResolver}, cfg.config().GetStringSlice(cfgResolveOrder)) + require.Equal(t, 15*time.Second, cfg.config().GetDuration(cfgConTimeout)) +} + +func TestSetTLSEnabled(t *testing.T) { + cfg := settings() + + require.NoError(t, cfg.flags.Parse([]string{"--" + cfgTLSCertFile, "tls.crt", "--" + cfgTLSKeyFile, "tls.key"})) + require.NoError(t, cfg.reload()) + + require.True(t, cfg.config().GetBool(cfgServer+".0."+cfgTLSEnabled)) +} From 8362cd696e95d1e3b3d2bcce83deec08b0895918 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Tue, 28 Jan 2025 13:42:48 +0300 Subject: [PATCH 51/86] [#199] Port release v0.32.1 changelog Signed-off-by: Marina Biryukova --- CHANGELOG.md | 8 +++++++- VERSION | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd37815..ee0659c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ This document outlines major changes between releases. ### Added - Add handling quota limit reached error (#187) +## [0.32.1] - 2025-01-27 + +### Fixed +- SIGHUP panic (#198) + ## [0.32.0] - Khumbu - 2024-12-20 ### Fixed @@ -187,4 +192,5 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.2...v0.30.3 [0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.3...v0.31.0 [0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...v0.32.0 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.0...master \ No newline at end of file +[0.32.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.0...v0.32.1 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.1...master \ No newline at end of file diff --git a/VERSION b/VERSION index 420000f..0e22fde 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.32.0 +v0.32.1 From 72e5d645b9abb5647910ad2aa7ca48b01fe26037 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 27 Jan 2025 14:23:31 +0300 Subject: [PATCH 52/86] [#194] Fix updateServers finding logic Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index ac20d29..1bacbed 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -894,8 +894,8 @@ func (a *app) updateServers() error { if err := ser.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { return fmt.Errorf("failed to update tls certs: %w", err) } - found = true } + found = true } else if unbind := a.updateUnbindServerInfo(serverInfo); unbind { found = true } From 7e48ca626e129ec1526602fd6fc786d41dfeb44d Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 30 Jan 2025 16:38:02 +0300 Subject: [PATCH 53/86] [#202] Bump SDK version to the latest master Contains fixes: - memory leak in gRPC client, - panic and deadlock in tree pool. Signed-off-by: Alex Vanin --- go.mod | 22 ++++++++++------------ go.sum | 44 ++++++++++++++++++++++++-------------------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 46fe3bc..d0bfcbb 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 @@ -22,13 +22,13 @@ require ( github.com/testcontainers/testcontainers-go v0.35.0 github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/valyala/fasthttp v1.34.0 - go.opentelemetry.io/otel v1.28.0 - go.opentelemetry.io/otel/trace v1.28.0 + go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/net v0.26.0 + golang.org/x/net v0.30.0 golang.org/x/sys v0.28.0 - google.golang.org/grpc v1.66.2 + google.golang.org/grpc v1.69.2 ) require ( @@ -68,12 +68,10 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/ipfs/go-cid v0.0.7 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -122,8 +120,8 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.31.0 // indirect @@ -131,9 +129,9 @@ require ( golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/protobuf v1.36.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect diff --git a/go.sum b/go.sum index e5bfc09..8eb4ea9 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae h1:7gvuOTmS3oaOM79JkHWWlsvGqIRqsum5KnOI1TYqfn0= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae/go.mod h1:dbWUc5jOBTXVvssCLCYxkkSTL9jgLr1KruGP2FMAfiM= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -172,6 +172,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -418,8 +420,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= @@ -428,12 +430,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMey go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -528,8 +532,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -745,10 +749,10 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -765,8 +769,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= -google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -777,8 +781,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 1779593f46580a3d4ad11009f3a15e7472e99719 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 30 Jan 2025 16:39:27 +0300 Subject: [PATCH 54/86] [#203] Port changelog from support branch Signed-off-by: Alex Vanin --- CHANGELOG.md | 8 +++++++- VERSION | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee0659c..30fcf7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ This document outlines major changes between releases. ### Added - Add handling quota limit reached error (#187) +## [0.32.2] - 2025-02-03 + +### Fixed +- Possible memory leak in gRPC client (#202) + ## [0.32.1] - 2025-01-27 ### Fixed @@ -193,4 +198,5 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.3...v0.31.0 [0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...v0.32.0 [0.32.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.0...v0.32.1 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.1...master \ No newline at end of file +[0.32.2]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.1...v0.32.2 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.2...master \ No newline at end of file diff --git a/VERSION b/VERSION index 0e22fde..c6a2605 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.32.1 +v0.32.2 From 76bd6ea40f9ac78fb5e83f0fd20e116866aba7ec Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 7 Feb 2025 12:58:01 +0300 Subject: [PATCH 55/86] [#206] Bump go version in vulncheck go1.22.11 triggers GO-2025-3447 but this is applicable only for ppc64le platform. Signed-off-by: Alex Vanin --- .forgejo/workflows/vulncheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 529e8a8..5cb6e73 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -16,7 +16,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22.11' + go-version: '1.22.12' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest From 6a4d3206bd18ba9b1881c1f0f4be3aafd9979554 Mon Sep 17 00:00:00 2001 From: Aleksey Kravchenko Date: Tue, 28 Jan 2025 14:42:40 +0300 Subject: [PATCH 56/86] [#195] Add tags support Signed-off-by: Aleksey Kravchenko --- cmd/http-gw/app.go | 203 ++++++++++++++++++++--------- cmd/http-gw/logger.go | 174 +++++++++++++++++++++++++ cmd/http-gw/settings.go | 165 +++++++---------------- config/config.env | 2 + config/config.yaml | 4 + docs/gate-configuration.md | 29 +++++ internal/cache/buckets.go | 4 +- internal/cache/netmap.go | 2 +- internal/handler/browse.go | 6 +- internal/handler/download.go | 27 ++-- internal/handler/filter.go | 3 +- internal/handler/handler.go | 26 ++-- internal/handler/head.go | 6 +- internal/handler/multipart.go | 6 +- internal/handler/multipart_test.go | 5 +- internal/handler/reader.go | 5 +- internal/handler/upload.go | 27 ++-- internal/handler/utils.go | 4 +- internal/logs/logs.go | 149 ++++++++++++--------- internal/net/event_handler.go | 6 +- internal/service/frostfs/source.go | 2 +- metrics/service.go | 12 +- 22 files changed, 572 insertions(+), 295 deletions(-) create mode 100644 cmd/http-gw/logger.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 1bacbed..103c72b 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -44,6 +44,7 @@ import ( "github.com/valyala/fasthttp" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "golang.org/x/exp/slices" ) @@ -51,7 +52,6 @@ type ( app struct { ctx context.Context log *zap.Logger - logLevel zap.AtomicLevel pool *pool.Pool treePool *treepool.Pool key *keys.PrivateKey @@ -94,6 +94,7 @@ type ( reconnectInterval time.Duration dialerSource *internalnet.DialerSource workerPoolSize int + logLevelConfig *logLevelConfig mu sync.RWMutex defaultTimestamp bool @@ -113,6 +114,15 @@ type ( enableFilepathFallback bool } + tagsConfig struct { + tagLogs sync.Map + } + + logLevelConfig struct { + logLevel zap.AtomicLevel + tagsConfig *tagsConfig + } + CORS struct { AllowOrigin string AllowMethods []string @@ -123,14 +133,91 @@ type ( } ) +func newLogLevel(v *viper.Viper) zap.AtomicLevel { + ll, err := getLogLevel(v) + if err != nil { + panic(err.Error()) + } + atomicLogLevel := zap.NewAtomicLevel() + atomicLogLevel.SetLevel(ll) + return atomicLogLevel +} + +func newTagsConfig(v *viper.Viper, ll zapcore.Level) *tagsConfig { + var t tagsConfig + if err := t.update(v, ll); err != nil { + // panic here is analogue of the similar panic during common log level initialization. + panic(err.Error()) + } + return &t +} + +func newLogLevelConfig(lvl zap.AtomicLevel, tagsConfig *tagsConfig) *logLevelConfig { + return &logLevelConfig{ + logLevel: lvl, + tagsConfig: tagsConfig, + } +} + +func (l *logLevelConfig) update(cfg *viper.Viper, log *zap.Logger) { + if lvl, err := getLogLevel(cfg); err != nil { + log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) + } else { + l.logLevel.SetLevel(lvl) + } + + if err := l.tagsConfig.update(cfg, l.logLevel.Level()); err != nil { + log.Warn(logs.TagsLogConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) + } +} + +func (t *tagsConfig) LevelEnabled(tag string, tgtLevel zapcore.Level) bool { + lvl, ok := t.tagLogs.Load(tag) + if !ok { + return false + } + + return lvl.(zapcore.Level).Enabled(tgtLevel) +} + +func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error { + tags, err := fetchLogTagsConfig(cfg, ll) + if err != nil { + return err + } + + t.tagLogs.Range(func(key, value any) bool { + k := key.(string) + v := value.(zapcore.Level) + + if lvl, ok := tags[k]; ok { + if lvl != v { + t.tagLogs.Store(key, lvl) + } + } else { + t.tagLogs.Delete(key) + delete(tags, k) + } + return true + }) + + for k, v := range tags { + t.tagLogs.Store(k, v) + } + + return nil +} + func newApp(ctx context.Context, cfg *appCfg) App { logSettings := &loggerSettings{} - log := pickLogger(cfg.config(), logSettings) + logLevel := newLogLevel(cfg.config()) + tagConfig := newTagsConfig(cfg.config(), logLevel.Level()) + logConfig := newLogLevelConfig(logLevel, tagConfig) + log := pickLogger(cfg.config(), logConfig.logLevel, logSettings, tagConfig) a := &app{ ctx: ctx, log: log.logger, - logLevel: log.lvl, cfg: cfg, loggerSettings: logSettings, webServer: new(fasthttp.Server), @@ -138,7 +225,7 @@ func newApp(ctx context.Context, cfg *appCfg) App { bucketCache: cache.NewBucketCache(getBucketCacheOptions(cfg.config(), log.logger), cfg.config().GetBool(cfgFeaturesTreePoolNetmapSupport)), } - a.initAppSettings() + a.initAppSettings(logConfig) // -- setup FastHTTP server -- a.webServer.Name = "frost-http-gw" @@ -172,11 +259,12 @@ func (a *app) config() *viper.Viper { return a.cfg.config() } -func (a *app) initAppSettings() { +func (a *app) initAppSettings(lc *logLevelConfig) { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.config()), dialerSource: getDialerSource(a.log, a.config()), workerPoolSize: a.config().GetInt(cfgWorkerPoolSize), + logLevelConfig: lc, } a.settings.update(a.config(), a.log) } @@ -324,7 +412,7 @@ func (a *app) initResolver() { var err error a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) if err != nil { - a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err), logs.TagField(logs.TagApp)) } } @@ -338,11 +426,12 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) { order := a.config().GetStringSlice(cfgResolveOrder) if resolveCfg.RPCAddress == "" { order = remove(order, resolver.NNSResolver) - a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided) + a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided, logs.TagField(logs.TagApp)) } if len(order) == 0 { - a.log.Info(logs.ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty) + a.log.Info(logs.ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty, + logs.TagField(logs.TagApp)) } return order, resolveCfg @@ -357,7 +446,7 @@ func (a *app) initMetrics() { func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics { if !enabled { - logger.Warn(logs.MetricsAreDisabled) + logger.Warn(logs.MetricsAreDisabled, logs.TagField(logs.TagApp)) } return &gateMetrics{ logger: logger, @@ -375,7 +464,7 @@ func (m *gateMetrics) isEnabled() bool { func (m *gateMetrics) SetEnabled(enabled bool) { if !enabled { - m.logger.Warn(logs.MetricsAreDisabled) + m.logger.Warn(logs.MetricsAreDisabled, logs.TagField(logs.TagApp)) } m.mu.Lock() @@ -438,7 +527,7 @@ func getFrostFSKey(cfg *viper.Viper, log *zap.Logger) (*keys.PrivateKey, error) walletPath := cfg.GetString(cfgWalletPath) if len(walletPath) == 0 { - log.Info(logs.NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun) + log.Info(logs.NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun, logs.TagField(logs.TagApp)) key, err := keys.NewPrivateKey() if err != nil { return nil, err @@ -495,7 +584,10 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*keys } func (a *app) Wait() { - a.log.Info(logs.StartingApplication, zap.String("app_name", "frostfs-http-gw"), zap.String("version", Version)) + a.log.Info(logs.StartingApplication, + zap.String("app_name", "frostfs-http-gw"), + zap.String("version", Version), + logs.TagField(logs.TagApp)) a.metrics.SetVersion(Version) a.setHealthStatus() @@ -526,10 +618,10 @@ func (a *app) Serve() { for i := range servs { go func(i int) { - a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address())) + a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address()), logs.TagField(logs.TagApp)) if err := a.webServer.Serve(servs[i].Listener()); err != nil && err != http.ErrServerClosed { a.metrics.MarkUnhealthy(servs[i].Address()) - a.log.Fatal(logs.ListenAndServe, zap.Error(err)) + a.log.Fatal(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp)) } }(i) } @@ -551,7 +643,7 @@ LOOP: } } - a.log.Info(logs.ShuttingDownWebServer, zap.Error(a.webServer.Shutdown())) + a.log.Info(logs.ShuttingDownWebServer, zap.Error(a.webServer.Shutdown()), logs.TagField(logs.TagApp)) a.metrics.Shutdown() a.stopServices() @@ -561,7 +653,7 @@ LOOP: func (a *app) initWorkerPool() *ants.Pool { workerPool, err := ants.NewPool(a.settings.workerPoolSize) if err != nil { - a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err), logs.TagField(logs.TagApp)) } return workerPool } @@ -572,37 +664,33 @@ func (a *app) shutdownTracing() { defer cancel() if err := tracing.Shutdown(shdnCtx); err != nil { - a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err)) + a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err), logs.TagField(logs.TagApp)) } } func (a *app) configReload(ctx context.Context) { - a.log.Info(logs.SIGHUPConfigReloadStarted) + a.log.Info(logs.SIGHUPConfigReloadStarted, logs.TagField(logs.TagApp)) if !a.config().IsSet(cmdConfig) && !a.config().IsSet(cmdConfigDir) { - a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed) + a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed, logs.TagField(logs.TagApp)) return } if err := a.cfg.reload(); err != nil { - a.log.Warn(logs.FailedToReloadConfig, zap.Error(err)) + a.log.Warn(logs.FailedToReloadConfig, zap.Error(err), logs.TagField(logs.TagApp)) return } - if lvl, err := getLogLevel(a.config()); err != nil { - a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err)) - } else { - a.logLevel.SetLevel(lvl) - } + a.settings.logLevelConfig.update(a.cfg.settings, a.log) if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil { - a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err)) + a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) } if err := a.resolver.UpdateResolvers(a.getResolverConfig()); err != nil { - a.log.Warn(logs.FailedToUpdateResolvers, zap.Error(err)) + a.log.Warn(logs.FailedToUpdateResolvers, zap.Error(err), logs.TagField(logs.TagApp)) } if err := a.updateServers(); err != nil { - a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err)) + a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err), logs.TagField(logs.TagApp)) } a.setRuntimeParameters() @@ -616,7 +704,7 @@ func (a *app) configReload(ctx context.Context) { a.initTracing(ctx) a.setHealthStatus() - a.log.Info(logs.SIGHUPConfigReloadCompleted) + a.log.Info(logs.SIGHUPConfigReloadCompleted, logs.TagField(logs.TagApp)) } func (a *app) startServices() { @@ -654,20 +742,20 @@ func (a *app) configureRouter(h *handler.Handler) { r.POST("/upload/{cid}", a.addMiddlewares(h.Upload)) r.OPTIONS("/upload/{cid}", a.addPreflight()) - a.log.Info(logs.AddedPathUploadCid) + a.log.Info(logs.AddedPathUploadCid, logs.TagField(logs.TagApp)) r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(h.DownloadByAddressOrBucketName)) r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(h.HeadByAddressOrBucketName)) r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight()) - a.log.Info(logs.AddedPathGetCidOid) + a.log.Info(logs.AddedPathGetCidOid, logs.TagField(logs.TagApp)) r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.DownloadByAttribute)) r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute)) r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) - a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) + a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal, logs.TagField(logs.TagApp)) r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZip)) r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadTar)) r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight()) - a.log.Info(logs.AddedPathZipCidPrefix) + a.log.Info(logs.AddedPathZipCidPrefix, logs.TagField(logs.TagApp)) a.webServer.Handler = r.Handler } @@ -756,14 +844,11 @@ func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { reqCtx = utils.SetReqLog(reqCtx, log) utils.SetContextToRequest(reqCtx, req) - fields := []zap.Field{ - zap.String("remote", req.RemoteAddr().String()), + log.Info(logs.Request, zap.String("remote", req.RemoteAddr().String()), zap.ByteString("method", req.Method()), zap.ByteString("path", req.Path()), zap.ByteString("query", req.QueryArgs().QueryString()), - } - - log.Info(logs.Request, fields...) + logs.TagField(logs.TagDatapath)) h(req) } } @@ -807,7 +892,7 @@ func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { if err != nil { log := utils.GetReqLogOrDefault(reqCtx, a.log) - log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err)) + log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err), logs.TagField(logs.TagDatapath)) handler.ResponseError(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -866,17 +951,17 @@ func (a *app) initServers(ctx context.Context) { if err != nil { a.unbindServers = append(a.unbindServers, serverInfo) a.metrics.MarkUnhealthy(serverInfo.Address) - a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...) + a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err), logs.TagField(logs.TagApp))...) continue } a.metrics.MarkHealthy(serverInfo.Address) a.servers = append(a.servers, srv) - a.log.Info(logs.AddServer, fields...) + a.log.Info(logs.AddServer, append(fields, logs.TagField(logs.TagApp))...) } if len(a.servers) == 0 { - a.log.Fatal(logs.NoHealthyServers) + a.log.Fatal(logs.NoHealthyServers, logs.TagField(logs.TagApp)) } } @@ -950,13 +1035,14 @@ func (a *app) initTracing(ctx context.Context) { if trustedCa := a.config().GetString(cfgTracingTrustedCa); trustedCa != "" { caBytes, err := os.ReadFile(trustedCa) if err != nil { - a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp)) return } certPool := x509.NewCertPool() ok := certPool.AppendCertsFromPEM(caBytes) if !ok { - a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert")) + a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert"), + logs.TagField(logs.TagApp)) return } cfg.ServerCaCertPool = certPool @@ -964,24 +1050,24 @@ func (a *app) initTracing(ctx context.Context) { attributes, err := fetchTracingAttributes(a.config()) if err != nil { - a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp)) return } cfg.Attributes = attributes updated, err := tracing.Setup(ctx, cfg) if err != nil { - a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp)) } if updated { - a.log.Info(logs.TracingConfigUpdated) + a.log.Info(logs.TracingConfigUpdated, logs.TagField(logs.TagApp)) } } func (a *app) setRuntimeParameters() { if len(os.Getenv("GOMEMLIMIT")) != 0 { // default limit < yaml limit < app env limit < GOMEMLIMIT - a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) + a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT, logs.TagField(logs.TagApp)) return } @@ -990,7 +1076,8 @@ func (a *app) setRuntimeParameters() { if softMemoryLimit != previous { a.log.Info(logs.RuntimeSoftMemoryLimitUpdated, zap.Int64("new_value", softMemoryLimit), - zap.Int64("old_value", previous)) + zap.Int64("old_value", previous), + logs.TagField(logs.TagApp)) } } @@ -1016,34 +1103,32 @@ func (a *app) tryReconnect(ctx context.Context, sr *fasthttp.Server) bool { a.mu.Lock() defer a.mu.Unlock() - a.log.Info(logs.ServerReconnecting) + a.log.Info(logs.ServerReconnecting, logs.TagField(logs.TagApp)) var failedServers []ServerInfo for _, serverInfo := range a.unbindServers { - fields := []zap.Field{ - zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled), - zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile), - } - srv, err := newServer(ctx, serverInfo) if err != nil { - a.log.Warn(logs.ServerReconnectFailed, zap.Error(err)) + a.log.Warn(logs.ServerReconnectFailed, zap.Error(err), logs.TagField(logs.TagApp)) failedServers = append(failedServers, serverInfo) a.metrics.MarkUnhealthy(serverInfo.Address) continue } go func() { - a.log.Info(logs.StartingServer, zap.String("address", srv.Address())) + a.log.Info(logs.StartingServer, zap.String("address", srv.Address()), logs.TagField(logs.TagApp)) a.metrics.MarkHealthy(serverInfo.Address) if err = sr.Serve(srv.Listener()); err != nil && !errors.Is(err, http.ErrServerClosed) { - a.log.Warn(logs.ListenAndServe, zap.Error(err)) + a.log.Warn(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp)) a.metrics.MarkUnhealthy(serverInfo.Address) } }() a.servers = append(a.servers, srv) - a.log.Info(logs.ServerReconnectedSuccessfully, fields...) + a.log.Info(logs.ServerReconnectedSuccessfully, + zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled), + zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile), + logs.TagField(logs.TagApp)) } a.unbindServers = failedServers diff --git a/cmd/http-gw/logger.go b/cmd/http-gw/logger.go new file mode 100644 index 0000000..91105f7 --- /dev/null +++ b/cmd/http-gw/logger.go @@ -0,0 +1,174 @@ +package main + +import ( + "fmt" + "os" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/zapjournald" + "github.com/spf13/viper" + "github.com/ssgreg/journald" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func getLogLevel(v *viper.Viper) (zapcore.Level, error) { + var lvl zapcore.Level + lvlStr := v.GetString(cfgLoggerLevel) + err := lvl.UnmarshalText([]byte(lvlStr)) + if err != nil { + return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+ + "value should be one of %v", lvlStr, err, [...]zapcore.Level{ + zapcore.DebugLevel, + zapcore.InfoLevel, + zapcore.WarnLevel, + zapcore.ErrorLevel, + zapcore.DPanicLevel, + zapcore.PanicLevel, + zapcore.FatalLevel, + }) + } + return lvl, nil +} + +var _ zapcore.Core = (*zapCoreTagFilterWrapper)(nil) + +type zapCoreTagFilterWrapper struct { + core zapcore.Core + settings TagFilterSettings + extra []zap.Field +} + +type TagFilterSettings interface { + LevelEnabled(tag string, lvl zapcore.Level) bool +} + +func (c *zapCoreTagFilterWrapper) Enabled(level zapcore.Level) bool { + return c.core.Enabled(level) +} + +func (c *zapCoreTagFilterWrapper) With(fields []zapcore.Field) zapcore.Core { + return &zapCoreTagFilterWrapper{ + core: c.core.With(fields), + settings: c.settings, + extra: append(c.extra, fields...), + } +} + +func (c *zapCoreTagFilterWrapper) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.core.Enabled(entry.Level) { + return checked.AddCore(entry, c) + } + return checked +} + +func (c *zapCoreTagFilterWrapper) Write(entry zapcore.Entry, fields []zapcore.Field) error { + if c.shouldSkip(entry, fields) || c.shouldSkip(entry, c.extra) { + return nil + } + + return c.core.Write(entry, fields) +} + +func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field) bool { + for _, field := range fields { + if field.Key == logs.TagFieldName && field.Type == zapcore.StringType { + if !c.settings.LevelEnabled(field.String, entry.Level) { + return true + } + break + } + } + + return false +} + +func (c *zapCoreTagFilterWrapper) Sync() error { + return c.core.Sync() +} + +func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) zapcore.Core { + core = &zapCoreTagFilterWrapper{ + core: core, + settings: tagSetting, + } + + if v.GetBool(cfgLoggerSamplingEnabled) { + core = zapcore.NewSamplerWithOptions(core, + v.GetDuration(cfgLoggerSamplingInterval), + v.GetInt(cfgLoggerSamplingInitial), + v.GetInt(cfgLoggerSamplingThereafter), + zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) { + if dec&zapcore.LogDropped > 0 { + loggerSettings.DroppedLogsInc() + } + })) + } + + return core +} + +func newLogEncoder() zapcore.Encoder { + c := zap.NewProductionEncoderConfig() + c.EncodeTime = zapcore.ISO8601TimeEncoder + + return zapcore.NewConsoleEncoder(c) +} + +// newStdoutLogger constructs a zap.Logger instance for current application. +// Panics on failure. +// +// Logger is built from zap's production logging configuration with: +// - parameterized level (debug by default) +// - console encoding +// - ISO8601 time encoding +// +// Logger records a stack trace for all messages at or above fatal level. +// +// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. +func newStdoutLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) *Logger { + stdout := zapcore.AddSync(os.Stderr) + + consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, lvl) + consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, loggerSettings, tagSetting) + + return &Logger{ + logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), + lvl: lvl, + } +} + +func newJournaldLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) *Logger { + encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) + + core := zapjournald.NewCore(lvl, encoder, &journald.Journal{}, zapjournald.SyslogFields) + coreWithContext := core.With([]zapcore.Field{ + zapjournald.SyslogFacility(zapjournald.LogDaemon), + zapjournald.SyslogIdentifier(), + zapjournald.SyslogPid(), + }) + + coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, loggerSettings, tagSetting) + + return &Logger{ + logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), + lvl: lvl, + } +} + +type LoggerAppSettings interface { + DroppedLogsInc() +} + +func pickLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger { + dest := v.GetString(cfgLoggerDestination) + + switch dest { + case destinationStdout: + return newStdoutLogger(v, lvl, loggerSettings, tagSettings) + case destinationJournald: + return newJournaldLogger(v, lvl, loggerSettings, tagSettings) + default: + panic(fmt.Sprintf("wrong destination for logger: %s", dest)) + } +} diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 5670f87..d8331ad 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -23,10 +23,8 @@ import ( grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" - "git.frostfs.info/TrueCloudLab/zapjournald" "github.com/spf13/pflag" "github.com/spf13/viper" - "github.com/ssgreg/journald" "github.com/valyala/fasthttp" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -111,6 +109,11 @@ const ( cfgLoggerSamplingThereafter = "logger.sampling.thereafter" cfgLoggerSamplingInterval = "logger.sampling.interval" + cfgLoggerTags = "logger.tags" + cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d." + cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "name" + cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level" + // Wallet. cfgWalletPassphrase = "wallet.passphrase" cfgWalletPath = "wallet.path" @@ -193,6 +196,8 @@ var ignore = map[string]struct{}{ cmdVersion: {}, } +var defaultTags = []string{logs.TagApp, logs.TagDatapath} + type Logger struct { logger *zap.Logger lvl zap.AtomicLevel @@ -499,112 +504,33 @@ func mergeConfig(v *viper.Viper, fileName string) error { return v.MergeConfig(cfgFile) } -type LoggerAppSettings interface { - DroppedLogsInc() -} +func fetchLogTagsConfig(v *viper.Viper, defaultLvl zapcore.Level) (map[string]zapcore.Level, error) { + res := make(map[string]zapcore.Level) -func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger { - lvl, err := getLogLevel(v) - if err != nil { - panic(err) + for i := 0; ; i++ { + name := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i)) + if name == "" { + break + } + + lvl := defaultLvl + level := v.GetString(fmt.Sprintf(cfgLoggerTagsLevelTmpl, i)) + if level != "" { + if err := lvl.Set(level); err != nil { + return nil, fmt.Errorf("failed to parse log tags config, unknown level: '%s'", level) + } + } + + res[name] = lvl } - dest := v.GetString(cfgLoggerDestination) - - switch dest { - case destinationStdout: - return newStdoutLogger(v, lvl, settings) - case destinationJournald: - return newJournaldLogger(v, lvl, settings) - default: - panic(fmt.Sprintf("wrong destination for logger: %s", dest)) - } -} - -// newStdoutLogger constructs a zap.Logger instance for current application. -// Panics on failure. -// -// Logger is built from zap's production logging configuration with: -// - parameterized level (debug by default) -// - console encoding -// - ISO8601 time encoding -// -// Logger records a stack trace for all messages at or above fatal level. -// -// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. -func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger { - stdout := zapcore.AddSync(os.Stderr) - level := zap.NewAtomicLevelAt(lvl) - - consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level) - consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, settings) - - return &Logger{ - logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), - lvl: level, - } -} - -func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger { - level := zap.NewAtomicLevelAt(lvl) - - encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) - - core := zapjournald.NewCore(level, encoder, &journald.Journal{}, zapjournald.SyslogFields) - coreWithContext := core.With([]zapcore.Field{ - zapjournald.SyslogFacility(zapjournald.LogDaemon), - zapjournald.SyslogIdentifier(), - zapjournald.SyslogPid(), - }) - - coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, settings) - - return &Logger{ - logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), - lvl: level, - } -} - -func newLogEncoder() zapcore.Encoder { - c := zap.NewProductionEncoderConfig() - c.EncodeTime = zapcore.ISO8601TimeEncoder - - return zapcore.NewConsoleEncoder(c) -} - -func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, settings LoggerAppSettings) zapcore.Core { - if v.GetBool(cfgLoggerSamplingEnabled) { - core = zapcore.NewSamplerWithOptions(core, - v.GetDuration(cfgLoggerSamplingInterval), - v.GetInt(cfgLoggerSamplingInitial), - v.GetInt(cfgLoggerSamplingThereafter), - zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) { - if dec&zapcore.LogDropped > 0 { - settings.DroppedLogsInc() - } - })) + if len(res) == 0 && !v.IsSet(cfgLoggerTags) { + for _, tag := range defaultTags { + res[tag] = defaultLvl + } } - return core -} - -func getLogLevel(v *viper.Viper) (zapcore.Level, error) { - var lvl zapcore.Level - lvlStr := v.GetString(cfgLoggerLevel) - err := lvl.UnmarshalText([]byte(lvlStr)) - if err != nil { - return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+ - "value should be one of %v", lvlStr, err, [...]zapcore.Level{ - zapcore.DebugLevel, - zapcore.InfoLevel, - zapcore.WarnLevel, - zapcore.ErrorLevel, - zapcore.DPanicLevel, - zapcore.PanicLevel, - zapcore.FatalLevel, - }) - } - return lvl, nil + return res, nil } func fetchReconnectInterval(cfg *viper.Viper) time.Duration { @@ -620,20 +546,19 @@ func fetchIndexPageTemplate(v *viper.Viper, l *zap.Logger) (string, bool) { if !v.GetBool(cfgIndexPageEnabled) { return "", false } - reader, err := os.Open(v.GetString(cfgIndexPageTemplatePath)) if err != nil { - l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err), logs.TagField(logs.TagApp)) return "", true } tmpl, err := io.ReadAll(reader) if err != nil { - l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err), logs.TagField(logs.TagApp)) return "", true } - l.Info(logs.SetCustomIndexPageTemplate) + l.Info(logs.SetCustomIndexPageTemplate, logs.TagField(logs.TagApp)) return string(tmpl), true } @@ -674,7 +599,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { } if _, ok := seen[serverInfo.Address]; ok { - log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address)) + log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address), logs.TagField(logs.TagApp)) continue } seen[serverInfo.Address] = struct{}{} @@ -687,7 +612,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { func (a *app) initPools(ctx context.Context) { key, err := getFrostFSKey(a.config(), a.log) if err != nil { - a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) + a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err), logs.TagField(logs.TagApp)) } var prm pool.InitParameters @@ -695,7 +620,8 @@ func (a *app) initPools(ctx context.Context) { prm.SetKey(&key.PrivateKey) prmTree.SetKey(key) - a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) + a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())), + logs.TagField(logs.TagApp)) for _, peer := range fetchPeers(a.log, a.config()) { prm.AddNode(peer) @@ -750,11 +676,11 @@ func (a *app) initPools(ctx context.Context) { p, err := pool.NewPool(prm) if err != nil { - a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err), logs.TagField(logs.TagApp)) } if err = p.Dial(ctx); err != nil { - a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err), logs.TagField(logs.TagApp)) } if a.config().GetBool(cfgFeaturesTreePoolNetmapSupport) { @@ -763,10 +689,10 @@ func (a *app) initPools(ctx context.Context) { treePool, err := treepool.NewPool(prmTree) if err != nil { - a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err), logs.TagField(logs.TagApp)) } if err = treePool.Dial(ctx); err != nil { - a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err), logs.TagField(logs.TagApp)) } a.pool = p @@ -797,7 +723,8 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam { l.Info(logs.AddedStoragePeer, zap.Int("priority", priority), zap.String("address", address), - zap.Float64("weight", weight)) + zap.Float64("weight", weight), + logs.TagField(logs.TagApp)) } return nodes @@ -836,7 +763,8 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV l.Error(logs.InvalidLifetimeUsingDefaultValue, zap.String("parameter", cfgEntry), zap.Duration("value in config", lifetime), - zap.Duration("default", defaultValue)) + zap.Duration("default", defaultValue), + logs.TagField(logs.TagApp)) } else { return lifetime } @@ -852,7 +780,8 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue l.Error(logs.InvalidCacheSizeUsingDefaultValue, zap.String("parameter", cfgEntry), zap.Int("value in config", size), - zap.Int("default", defaultValue)) + zap.Int("default", defaultValue), + logs.TagField(logs.TagApp)) } else { return size } @@ -864,7 +793,7 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource { source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger)) if err != nil { - logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err)) + logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err), logs.TagField(logs.TagApp)) } return source } diff --git a/config/config.env b/config/config.env index db619b5..af0eba1 100644 --- a/config/config.env +++ b/config/config.env @@ -20,6 +20,8 @@ HTTP_GW_LOGGER_SAMPLING_ENABLED=false HTTP_GW_LOGGER_SAMPLING_INITIAL=100 HTTP_GW_LOGGER_SAMPLING_THEREAFTER=100 HTTP_GW_LOGGER_SAMPLING_INTERVAL=1s +HTTP_GW_LOGGER_TAGS_0_NAME=app +HTTP_GW_LOGGER_TAGS_1_NAME=datapath HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443 HTTP_GW_SERVER_0_TLS_ENABLED=false diff --git a/config/config.yaml b/config/config.yaml index a70ec9a..8c51591 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -29,6 +29,10 @@ logger: initial: 100 thereafter: 100 interval: 1s + tags: + - name: app + - name: datapath + level: debug server: - address: 0.0.0.0:8080 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 6aadd1f..4f9bc3b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -174,6 +174,11 @@ logger: initial: 100 thereafter: 100 interval: 1s + tags: + - name: "app" + level: info + - name: "datapath" + - name: "external_storage_tree" ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -184,6 +189,30 @@ logger: | `sampling.initial` | `int` | no | '100' | Sampling count of first log entries. | | `sampling.thereafter` | `int` | no | '100' | Sampling count of entries after an `interval`. | | `sampling.interval` | `duration` | no | '1s' | Sampling interval of messaging similar entries. | +| `sampling.tags` | `[]Tag` | yes | | Tagged log entries that should be additionally logged (available tags see in the next section). | + +## Tags + +There are additional log entries that can hurt performance and can be additionally logged by using `logger.tags` +parameter. Available tags: + +```yaml +tags: + - name: "app" + level: info +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------| +| `name` | `string` | yes | | Tag name. Possible values see below in `Tag values` section. | +| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | + +### Tag values + +* `app` - common application logs (enabled by default). +* `datapath` - main logic of application (enabled by default). +* `external_storage` - external interaction with storage node. +* `external_storage_tree` - external interaction with tree service in storage node. # `web` section diff --git a/internal/cache/buckets.go b/internal/cache/buckets.go index 2fa8f25..91ae5b2 100644 --- a/internal/cache/buckets.go +++ b/internal/cache/buckets.go @@ -72,7 +72,7 @@ func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo { key, ok := entry.(string) if !ok { o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), - zap.String("expected", fmt.Sprintf("%T", key))) + zap.String("expected", fmt.Sprintf("%T", key)), logs.TagField(logs.TagDatapath)) return nil } @@ -88,7 +88,7 @@ func (o *BucketCache) get(key string) *data.BucketInfo { result, ok := entry.(*data.BucketInfo) if !ok { o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), - zap.String("expected", fmt.Sprintf("%T", result))) + zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath)) return nil } diff --git a/internal/cache/netmap.go b/internal/cache/netmap.go index 6d91fe7..ce01b47 100644 --- a/internal/cache/netmap.go +++ b/internal/cache/netmap.go @@ -53,7 +53,7 @@ func (c *NetmapCache) Get() *netmap.NetMap { result, ok := entry.(netmap.NetMap) if !ok { c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), - zap.String("expected", fmt.Sprintf("%T", result))) + zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath)) return nil } diff --git a/internal/handler/browse.go b/internal/handler/browse.go index 64ad1f5..2d0e34d 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -230,7 +230,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck } for objExt := range resp { if objExt.Error != nil { - log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error)) + log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error), logs.TagField(logs.TagExternalStorage)) result.hasErrors = true continue } @@ -273,7 +273,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re }) if err != nil { wg.Done() - log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err)) + log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath)) } select { case <-ctx.Done(): @@ -283,7 +283,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re } }) if err != nil { - log.Error(logs.FailedToIterateOverResponse, zap.Error(err)) + log.Error(logs.FailedToIterateOverResponse, zap.Error(err), logs.TagField(logs.TagDatapath)) } wg.Wait() }() diff --git a/internal/handler/download.go b/internal/handler/download.go index d5fac23..783fe09 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -43,6 +43,8 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + log.Error(logs.FailedToCheckIfSettingsNodeExist, zap.String("cid", bktInfo.CID.String()), + zap.Error(checkS3Err), logs.TagField(logs.TagExternalStorageTree)) logAndSendBucketError(c, log, checkS3Err) return } @@ -121,13 +123,13 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res }), ) if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) + log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) return } else if objectsWritten == 0 { - log.Warn(logs.ObjectsNotFound) + log.Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) } if err := zipWriter.Close(); err != nil { - log.Error(logs.CloseZipWriter, zap.Error(err)) + log.Error(logs.CloseZipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } } } @@ -187,10 +189,10 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res defer func() { if err := tarWriter.Close(); err != nil { - log.Error(logs.CloseTarWriter, zap.Error(err)) + log.Error(logs.CloseTarWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } if err := gzipWriter.Close(); err != nil { - log.Error(logs.CloseGzipWriter, zap.Error(err)) + log.Error(logs.CloseGzipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } }() @@ -204,9 +206,9 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res }), ) if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) + log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) } else if objectsWritten == 0 { - log.Warn(logs.ObjectsNotFound) + log.Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) } } } @@ -237,18 +239,18 @@ func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID resGet, err := h.frostfs.GetObject(ctx, prm) if err != nil { - log.Error(logs.FailedToGetObject, zap.Error(err)) + log.Error(logs.FailedToGetObject, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return false } fileWriter, err := createArchiveHeader(&resGet.Header) if err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.Error(err)) + log.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) return false } if err = writeToArchive(resGet, fileWriter, buf); err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.Error(err)) + log.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) return false } @@ -264,7 +266,8 @@ func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, prefix, err := url.QueryUnescape(prefix) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), + zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) return nil, err } @@ -273,7 +276,7 @@ func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, resSearch, err := h.search(ctx, cnrID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage)) ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return nil, err } diff --git a/internal/handler/filter.go b/internal/handler/filter.go index 745718a..da99db7 100644 --- a/internal/handler/filter.go +++ b/internal/handler/filter.go @@ -50,7 +50,8 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st l.Debug(logs.AddAttributeToResultObject, zap.String("key", k), - zap.String("val", v)) + zap.String("val", v), + logs.TagField(logs.TagDatapath)) }) return result, err diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4e23c3b..b27c607 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -206,11 +206,13 @@ func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) if err != nil { + log.Error(logs.FailedToGetLatestVersionOfObject, zap.Error(err), zap.String("cid", cnrID.String()), + zap.String("path", path), logs.TagField(logs.TagExternalStorageTree)) logAndSendBucketError(c, log, err) return } if foundOID.IsDeleteMarker { - log.Error(logs.ObjectWasDeleted) + log.Error(logs.ObjectWasDeleted, logs.TagField(logs.TagExternalStorageTree)) ResponseError(c, "object deleted", fasthttp.StatusNotFound) return } @@ -230,14 +232,16 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte key, err := url.QueryUnescape(key) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), + zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) return } val, err = url.QueryUnescape(val) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), + zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -271,7 +275,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { res, err := h.search(ctx, cnrID, attrKey, attrVal, object.MatchStringEqual) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("could not search for objects: %w", err) } defer res.Close() @@ -282,13 +286,13 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cn if n == 0 { switch { case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): - log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName) + log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) case errors.Is(err, io.EOF): - log.Error(logs.ObjectNotFound, zap.Error(err)) + log.Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) default: - log.Error(logs.ReadObjectListFailed, zap.Error(err)) + log.Error(logs.ReadObjectListFailed, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("read object list failed: %w", err) } } @@ -330,11 +334,16 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log * cnrID, err := h.resolveContainer(ctx, containerName) if err != nil { + log.Error(logs.CouldNotResolveContainerID, zap.Error(err), zap.String("cnrName", containerName), + logs.TagField(logs.TagDatapath)) return nil, err } bktInfo, err := h.readContainer(ctx, *cnrID) if err != nil { + log.Error(logs.CouldNotGetContainerInfo, zap.Error(err), zap.String("cnrName", containerName), + zap.String("cnrName", cnrID.String()), + logs.TagField(logs.TagExternalStorage)) return nil, err } @@ -342,7 +351,8 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log * log.Warn(logs.CouldntPutBucketIntoCache, zap.String("bucket name", bktInfo.Name), zap.Stringer("bucket cid", bktInfo.CID), - zap.Error(err)) + zap.Error(err), + logs.TagField(logs.TagDatapath)) } return bktInfo, nil diff --git a/internal/handler/head.go b/internal/handler/head.go index 94f5ccb..0b5adc4 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -67,7 +67,8 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid req.log.Info(logs.CouldntParseCreationDate, zap.String("key", key), zap.String("val", val), - zap.Error(err)) + zap.Error(err), + logs.TagField(logs.TagDatapath)) continue } req.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat)) @@ -131,6 +132,8 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + log.Error(logs.FailedToCheckIfSettingsNodeExist, zap.String("cid", bktInfo.CID.String()), + zap.Error(checkS3Err), logs.TagField(logs.TagExternalStorageTree)) logAndSendBucketError(c, log, checkS3Err) return } @@ -144,7 +147,6 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { logAndSendBucketError(c, log, checkS3Err) - return } } diff --git a/internal/handler/multipart.go b/internal/handler/multipart.go index ebf5edd..5ed2350 100644 --- a/internal/handler/multipart.go +++ b/internal/handler/multipart.go @@ -33,7 +33,7 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF name := part.FormName() if name == "" { - l.Debug(logs.IgnorePartEmptyFormName) + l.Debug(logs.IgnorePartEmptyFormName, logs.TagField(logs.TagDatapath)) continue } @@ -41,9 +41,9 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF // ignore multipart/form-data values if filename == "" { - l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name)) + l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name), logs.TagField(logs.TagDatapath)) if err = part.Close(); err != nil { - l.Warn(logs.FailedToCloseReader, zap.Error(err)) + l.Warn(logs.FailedToCloseReader, zap.Error(err), logs.TagField(logs.TagDatapath)) } continue } diff --git a/internal/handler/multipart_test.go b/internal/handler/multipart_test.go index 2c50a87..431d0d6 100644 --- a/internal/handler/multipart_test.go +++ b/internal/handler/multipart_test.go @@ -112,7 +112,7 @@ func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (Mul name := part.FormName() if name == "" { - l.Debug(logs.IgnorePartEmptyFormName) + l.Debug(logs.IgnorePartEmptyFormName, logs.TagField(logs.TagDatapath)) continue } @@ -120,8 +120,7 @@ func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (Mul // ignore multipart/form-data values if filename == "" { - l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name)) - + l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name), logs.TagField(logs.TagDatapath)) continue } diff --git a/internal/handler/reader.go b/internal/handler/reader.go index cbd8294..e8ac098 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -110,7 +110,8 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A if err = req.setTimestamp(val); err != nil { req.log.Error(logs.CouldntParseCreationDate, zap.String("val", val), - zap.Error(err)) + zap.Error(err), + logs.TagField(logs.TagDatapath)) } case object.AttributeContentType: contentType = val @@ -144,7 +145,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A return payload, nil }, filename) if err != nil && err != io.EOF { - req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) + req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index d1953c9..272e911 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -68,14 +68,14 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { - log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) + log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } filtered, err := filterHeaders(log, &c.Request.Header) if err != nil { - log.Error(logs.FailedToFilterHeaders, zap.Error(err)) + log.Error(logs.FailedToFilterHeaders, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } @@ -106,7 +106,7 @@ func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file Mul attributes, err := h.extractAttributes(c, log, filtered) if err != nil { - log.Error(logs.FailedToGetAttributes, zap.Error(err)) + log.Error(logs.FailedToGetAttributes, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -119,13 +119,14 @@ func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file Mul log.Debug(logs.ObjectUploaded, zap.String("oid", idObj.EncodeToString()), zap.String("FileName", file.FileName()), + logs.TagField(logs.TagExternalStorage), ) addr := newAddress(bkt.CID, idObj) c.Response.Header.SetContentType(jsonHeader) // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { - log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) + log.Error(logs.CouldNotEncodeResponse, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) return } @@ -162,13 +163,14 @@ func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, fil now := time.Now() if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { - log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err)) + log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err), + logs.TagField(logs.TagDatapath)) } else { now = parsed } } if err := utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { - log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) + log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err), logs.TagField(logs.TagDatapath)) return nil, err } attributes := make([]object.Attribute, 0, len(filtered)) @@ -205,7 +207,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read commonAttributes, err := h.extractAttributes(c, log, filtered) if err != nil { - log.Error(logs.FailedToGetAttributes, zap.Error(err)) + log.Error(logs.FailedToGetAttributes, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -213,16 +215,16 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read reader := file if bytes.EqualFold(c.Request.Header.Peek(fasthttp.HeaderContentEncoding), []byte("gzip")) { - log.Debug(logs.GzipReaderSelected) + log.Debug(logs.GzipReaderSelected, logs.TagField(logs.TagDatapath)) gzipReader, err := gzip.NewReader(file) if err != nil { - log.Error(logs.FailedToCreateGzipReader, zap.Error(err)) + log.Error(logs.FailedToCreateGzipReader, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could read gzip file: "+err.Error(), fasthttp.StatusBadRequest) return } defer func() { if err := gzipReader.Close(); err != nil { - log.Warn(logs.FailedToCloseReader, zap.Error(err)) + log.Warn(logs.FailedToCloseReader, zap.Error(err), logs.TagField(logs.TagDatapath)) } }() reader = gzipReader @@ -234,7 +236,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read if errors.Is(err, io.EOF) { break } else if err != nil { - log.Error(logs.FailedToReadFileFromTar, zap.Error(err)) + log.Error(logs.FailedToReadFileFromTar, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not get next entry: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -258,6 +260,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read log.Debug(logs.ObjectUploaded, zap.String("oid", idObj.EncodeToString()), zap.String("FileName", fileName), + logs.TagField(logs.TagExternalStorage), ) } } @@ -266,7 +269,7 @@ func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *za statusCode, msg, additionalFields := formErrorResponse("could not store file in frostfs", err) logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) - log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) + log.Error(logs.CouldNotStoreFileInFrostfs, append(logFields, logs.TagField(logs.TagExternalStorage))...) ResponseError(r, msg, statusCode) } diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 74932f3..0a1dc62 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -39,7 +39,7 @@ func (r *request) handleFrostFSErr(err error, start time.Time) { statusCode, msg, additionalFields := formErrorResponse("could not receive object", err) logFields = append(logFields, additionalFields...) - r.log.Error(logs.CouldNotReceiveObject, logFields...) + r.log.Error(logs.CouldNotReceiveObject, append(logFields, logs.TagField(logs.TagExternalStorage))...) ResponseError(r.RequestCtx, msg, statusCode) } @@ -85,7 +85,7 @@ func isValidValue(s string) bool { } func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { - log.Error(logs.CouldntGetBucket, zap.Error(err)) + log.Error(logs.CouldNotGetBucket, zap.Error(err), logs.TagField(logs.TagDatapath)) if client.IsErrContainerNotFound(err) { ResponseError(c, "Not Found", fasthttp.StatusNotFound) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 5f60b9b..072e9c9 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -1,63 +1,43 @@ package logs +import "go.uber.org/zap" + +const ( + TagFieldName = "tag" + + TagApp = "app" + TagDatapath = "datapath" + TagExternalStorage = "external_storage" + TagExternalStorageTree = "external_storage_tree" +) + +func TagField(tag string) zap.Field { + return zap.String(TagFieldName, tag) +} + +// Log messages with the "app" tag. const ( - CouldntParseCreationDate = "couldn't parse creation date" - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" - CouldNotReceiveObject = "could not receive object" - ObjectWasDeleted = "object was deleted" - CouldNotSearchForObjects = "could not search for objects" - ObjectNotFound = "object not found" - ReadObjectListFailed = "read object list failed" - FailedToAddObjectToArchive = "failed to add object to archive" - FailedToGetObject = "failed to get object" - IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" - ObjectsNotFound = "objects not found" - CloseZipWriter = "close zip writer" ServiceIsRunning = "service is running" ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" ShuttingDownService = "shutting down service" CantShutDownService = "can't shut down service" CantGracefullyShutDownService = "can't gracefully shut down service, force stop" - IgnorePartEmptyFormName = "ignore part, empty form name" - IgnorePartEmptyFilename = "ignore part, empty filename" - CouldNotReceiveMultipartForm = "could not receive multipart/form" - CouldNotParseClientTime = "could not parse client time" - CouldNotPrepareExpirationHeader = "could not prepare expiration header" - CouldNotEncodeResponse = "could not encode response" - CouldNotStoreFileInFrostfs = "could not store file in frostfs" - AddAttributeToResultObject = "add attribute to result object" FailedToCreateResolver = "failed to create resolver" FailedToCreateWorkerPool = "failed to create worker pool" - FailedToReadIndexPageTemplate = "failed to read index page template" - SetCustomIndexPageTemplate = "set custom index page template" - ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" - MetricsAreDisabled = "metrics are disabled" - NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" StartingApplication = "starting application" StartingServer = "starting server" ListenAndServe = "listen and serve" ShuttingDownWebServer = "shutting down web server" FailedToShutdownTracing = "failed to shutdown tracing" - SIGHUPConfigReloadStarted = "SIGHUP config reload started" - FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" - FailedToReloadConfig = "failed to reload config" - LogLevelWontBeUpdated = "log level won't be updated" - FailedToUpdateResolvers = "failed to update resolvers" - FailedToReloadServerParameters = "failed to reload server parameters" - SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" AddedPathUploadCid = "added path /upload/{cid}" AddedPathGetCidOid = "added path /get/{cid}/{oid}" AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" - Request = "request" - CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" FailedToAddServer = "failed to add server" AddServer = "add server" NoHealthyServers = "no healthy servers" FailedToInitializeTracing = "failed to initialize tracing" - TracingConfigUpdated = "tracing config updated" - ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" @@ -66,33 +46,86 @@ const ( FailedToDialConnectionPool = "failed to dial connection pool" FailedToCreateTreePool = "failed to create tree pool" FailedToDialTreePool = "failed to dial tree pool" - AddedStoragePeer = "added storage peer" - CouldntGetBucket = "could not get bucket" - CouldntPutBucketIntoCache = "couldn't put bucket info into cache" - FailedToSumbitTaskToPool = "failed to submit task to pool" - FailedToHeadObject = "failed to head object" - FailedToIterateOverResponse = "failed to iterate over search response" - InvalidCacheEntryType = "invalid cache entry type" - InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" - InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" - FailedToUnescapeQuery = "failed to unescape query" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" - WarnDuplicateAddress = "duplicate address" + FailedToSumbitTaskToPool = "failed to submit task to pool" MultinetDialSuccess = "multinet dial successful" MultinetDialFail = "multinet dial failed" + ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" + MetricsAreDisabled = "metrics are disabled" + NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" + SIGHUPConfigReloadStarted = "SIGHUP config reload started" + FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" + FailedToReloadConfig = "failed to reload config" + FailedToUpdateResolvers = "failed to update resolvers" + FailedToReloadServerParameters = "failed to reload server parameters" + SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" + TracingConfigUpdated = "tracing config updated" + ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" + AddedStoragePeer = "added storage peer" + InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" + InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" + WarnDuplicateAddress = "duplicate address" FailedToLoadMultinetConfig = "failed to load multinet config" MultinetConfigWontBeUpdated = "multinet config won't be updated" - ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" - CouldntCacheNetmap = "couldn't cache netmap" - FailedToFilterHeaders = "failed to filter headers" - FailedToReadFileFromTar = "failed to read file from tar" - FailedToGetAttributes = "failed to get attributes" - ObjectUploaded = "object uploaded" - CloseGzipWriter = "close gzip writer" - CloseTarWriter = "close tar writer" - FailedToCloseReader = "failed to close reader" - FailedToCreateGzipReader = "failed to create gzip reader" - GzipReaderSelected = "gzip reader selected" + LogLevelWontBeUpdated = "log level won't be updated" + TagsLogConfigWontBeUpdated = "tags log config won't be updated" + FailedToReadIndexPageTemplate = "failed to read index page template" + SetCustomIndexPageTemplate = "set custom index page template" +) + +// Log messages with the "datapath" tag. +const ( + CouldntParseCreationDate = "couldn't parse creation date" + CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" + FailedToAddObjectToArchive = "failed to add object to archive" + CloseZipWriter = "close zip writer" + IgnorePartEmptyFormName = "ignore part, empty form name" + IgnorePartEmptyFilename = "ignore part, empty filename" + CouldNotParseClientTime = "could not parse client time" + CouldNotPrepareExpirationHeader = "could not prepare expiration header" + CouldNotEncodeResponse = "could not encode response" + AddAttributeToResultObject = "add attribute to result object" + Request = "request" + CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" + CouldntPutBucketIntoCache = "couldn't put bucket info into cache" + FailedToIterateOverResponse = "failed to iterate over search response" + InvalidCacheEntryType = "invalid cache entry type" + FailedToUnescapeQuery = "failed to unescape query" + CouldntCacheNetmap = "couldn't cache netmap" + FailedToCloseReader = "failed to close reader" + FailedToFilterHeaders = "failed to filter headers" + FailedToReadFileFromTar = "failed to read file from tar" + FailedToGetAttributes = "failed to get attributes" + CloseGzipWriter = "close gzip writer" + CloseTarWriter = "close tar writer" + FailedToCreateGzipReader = "failed to create gzip reader" + GzipReaderSelected = "gzip reader selected" + CouldNotReceiveMultipartForm = "could not receive multipart/form" + ObjectsNotFound = "objects not found" + IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" + CouldNotGetBucket = "could not get bucket" + CouldNotResolveContainerID = "could not resolve container id" +) + +// Log messages with the "external_storage" tag. +const ( + CouldNotReceiveObject = "could not receive object" + CouldNotSearchForObjects = "could not search for objects" + ObjectNotFound = "object not found" + ReadObjectListFailed = "read object list failed" + CouldNotStoreFileInFrostfs = "could not store file in frostfs" + FailedToHeadObject = "failed to head object" + ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" + FailedToGetObject = "failed to get object" + ObjectUploaded = "object uploaded" + CouldNotGetContainerInfo = "could not get container info" +) + +// Log messages with the "external_storage_tree" tag. +const ( + ObjectWasDeleted = "object was deleted" + FailedToGetLatestVersionOfObject = "failed to get latest version of object" + FailedToCheckIfSettingsNodeExist = "Failed to check if settings node exists" ) diff --git a/internal/net/event_handler.go b/internal/net/event_handler.go index 9520c01..2826d35 100644 --- a/internal/net/event_handler.go +++ b/internal/net/event_handler.go @@ -17,9 +17,11 @@ func (l LogEventHandler) DialPerformed(sourceIP net.Addr, _, address string, err sourceIPString = sourceIP.Network() + "://" + sourceIP.String() } if err == nil { - l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), zap.String("destination", address)) + l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), + zap.String("destination", address), logs.TagField(logs.TagApp)) } else { - l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), zap.String("destination", address), zap.Error(err)) + l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), + zap.String("destination", address), logs.TagField(logs.TagApp)) } } diff --git a/internal/service/frostfs/source.go b/internal/service/frostfs/source.go index de6c681..84f7b74 100644 --- a/internal/service/frostfs/source.go +++ b/internal/service/frostfs/source.go @@ -40,7 +40,7 @@ func (s *Source) NetMapSnapshot(ctx context.Context) (netmap.NetMap, error) { } if err = s.netmapCache.Put(netmapSnapshot); err != nil { - s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err)) + s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err), logs.TagField(logs.TagDatapath)) } return netmapSnapshot, nil diff --git a/metrics/service.go b/metrics/service.go index dea5ac0..e6b803b 100644 --- a/metrics/service.go +++ b/metrics/service.go @@ -25,24 +25,24 @@ type Config struct { // Start runs http service with the exposed endpoint on the configured port. func (ms *Service) Start() { if ms.enabled { - ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr)) + ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp)) err := ms.ListenAndServe() if err != nil && err != http.ErrServerClosed { - ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort) + ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort, logs.TagField(logs.TagApp)) } } else { - ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled) + ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled, logs.TagField(logs.TagApp)) } } // ShutDown stops the service. func (ms *Service) ShutDown(ctx context.Context) { - ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr)) + ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp)) err := ms.Shutdown(ctx) if err != nil { - ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err)) + ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err), logs.TagField(logs.TagApp)) if err = ms.Close(); err != nil { - ms.log.Panic(logs.CantShutDownService, zap.Error(err)) + ms.log.Panic(logs.CantShutDownService, zap.Error(err), logs.TagField(logs.TagApp)) } } } From 1e8fa19bb9ec13bd4184801ecae28983943f18ca Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 7 Feb 2025 13:55:17 +0300 Subject: [PATCH 57/86] [#195] Make all initial logging tags as default tags Signed-off-by: Alex Vanin --- cmd/http-gw/settings.go | 2 +- docs/gate-configuration.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index d8331ad..12d73d6 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -196,7 +196,7 @@ var ignore = map[string]struct{}{ cmdVersion: {}, } -var defaultTags = []string{logs.TagApp, logs.TagDatapath} +var defaultTags = []string{logs.TagApp, logs.TagDatapath, logs.TagExternalStorage, logs.TagExternalStorageTree} type Logger struct { logger *zap.Logger diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 4f9bc3b..191e9bb 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -211,8 +211,8 @@ tags: * `app` - common application logs (enabled by default). * `datapath` - main logic of application (enabled by default). -* `external_storage` - external interaction with storage node. -* `external_storage_tree` - external interaction with tree service in storage node. +* `external_storage` - external interaction with storage node (enabled by default). +* `external_storage_tree` - external interaction with tree service in storage node (enabled by default). # `web` section From c509ce0b28748e79c2442b5ef2e434232b84967d Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 7 Feb 2025 13:56:11 +0300 Subject: [PATCH 58/86] [#195] Fix log record grouping Signed-off-by: Alex Vanin --- internal/logs/logs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 072e9c9..f8f1da9 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -49,7 +49,6 @@ const ( ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" - FailedToSumbitTaskToPool = "failed to submit task to pool" MultinetDialSuccess = "multinet dial successful" MultinetDialFail = "multinet dial failed" ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" @@ -107,6 +106,7 @@ const ( IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" CouldNotGetBucket = "could not get bucket" CouldNotResolveContainerID = "could not resolve container id" + FailedToSumbitTaskToPool = "failed to submit task to pool" ) // Log messages with the "external_storage" tag. From 11846df2660e782f414bfd10edd3bdc61e2308b1 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 05:48:14 +0300 Subject: [PATCH 59/86] [#145] handler: Add spans to detail the trace Signed-off-by: Roman Loginov --- internal/handler/download.go | 20 +++++++++++++++++--- internal/handler/handler.go | 12 +++++++++++- internal/handler/head.go | 9 ++++++++- internal/handler/upload.go | 18 ++++++++++++++++-- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/internal/handler/download.go b/internal/handler/download.go index 783fe09..b398a54 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -16,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -25,11 +26,14 @@ import ( // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByAddressOrBucketName") + defer span.End() + utils.SetContextToRequest(ctx, c) + cidParam := c.UserValue("cid").(string) oidParam := c.UserValue("oid").(string) downloadParam := c.QueryArgs().GetBool("download") - ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log).With( zap.String("cid", cidParam), zap.String("oid", oidParam), @@ -67,6 +71,10 @@ func shouldDownload(oidParam string, downloadParam bool) bool { // DownloadByAttribute handles attribute-based download requests. func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByAttribute") + defer span.End() + utils.SetContextToRequest(ctx, c) + h.byAttribute(c, h.receiveFile) } @@ -88,9 +96,12 @@ func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op // DownloadZip handles zip by prefix requests. func (h *Handler) DownloadZip(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadZip") + defer span.End() + utils.SetContextToRequest(ctx, c) + scid, _ := c.UserValue("cid").(string) - ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { @@ -154,9 +165,12 @@ func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, // DownloadTar forms tar.gz from objects by prefix. func (h *Handler) DownloadTar(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadTar") + defer span.End() + utils.SetContextToRequest(ctx, c) + scid, _ := c.UserValue("cid").(string) - ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index b27c607..69aecbf 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -195,6 +196,9 @@ func New(params *AppParams, config Config, tree layer.TreeService, workerPool *a // byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) { + ctx, span := tracing.StartSpanFromContext(ctx, "handler.byNativeAddress") + defer span.End() + addr := newAddress(cnrID, objID) handler(ctx, req, addr) } @@ -202,6 +206,9 @@ func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID // byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that // resolves object address from S3-like path /. func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) { + ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path") + defer span.End() + c, log := req.RequestCtx, req.log foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) @@ -382,6 +389,10 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket } func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.browseIndex") + defer span.End() + utils.SetContextToRequest(ctx, c) + if !h.config.IndexPageEnabled() { c.SetStatusCode(fasthttp.StatusNotFound) return @@ -390,7 +401,6 @@ func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) { cidURLParam := c.UserValue("cid").(string) oidURLParam := c.UserValue("oid").(string) - ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) log := reqLog.With(zap.String("cid", cidURLParam), zap.String("oid", oidURLParam)) diff --git a/internal/handler/head.go b/internal/handler/head.go index 0b5adc4..7718c9c 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/valyala/fasthttp" @@ -116,10 +117,12 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) { // HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format. func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByAddressOrBucketName") + defer span.End() + cidParam, _ := c.UserValue("cid").(string) oidParam, _ := c.UserValue("oid").(string) - ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log).With( zap.String("cid", cidParam), zap.String("oid", oidParam), @@ -152,5 +155,9 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { // HeadByAttribute handles attribute-based head requests. func (h *Handler) HeadByAttribute(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByAttribute") + defer span.End() + utils.SetContextToRequest(ctx, c) + h.byAttribute(c, h.headObject) } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 272e911..48d0495 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -17,6 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -50,13 +51,16 @@ func (pr *putResponse) encode(w io.Writer) error { // Upload handles multipart upload request. func (h *Handler) Upload(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.Upload") + defer span.End() + utils.SetContextToRequest(ctx, c) + var file MultipartFile scid, _ := c.UserValue("cid").(string) bodyStream := c.RequestBodyStream() drainBuf := make([]byte, drainBufSize) - ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) log := reqLog.With(zap.String("cid", scid)) @@ -102,6 +106,11 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file MultipartFile, filtered map[string]string) { c, log := req.RequestCtx, req.log + + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.uploadSingleObject") + defer span.End() + utils.SetContextToRequest(ctx, c) + setIfNotExist(filtered, object.AttributeFileName, file.FileName()) attributes, err := h.extractAttributes(c, log, filtered) @@ -160,6 +169,7 @@ func (h *Handler) uploadObject(c *fasthttp.RequestCtx, bkt *data.BucketInfo, att } func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, filtered map[string]string) ([]object.Attribute, error) { + ctx := utils.GetContextFromRequest(c) now := time.Now() if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { @@ -169,7 +179,7 @@ func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, fil now = parsed } } - if err := utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { + if err := utils.PrepareExpirationHeader(ctx, h.frostfs, filtered, now); err != nil { log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err), logs.TagField(logs.TagDatapath)) return nil, err } @@ -200,6 +210,10 @@ func newAttribute(key string, val string) object.Attribute { func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.ReadCloser, filtered map[string]string) { c, log := req.RequestCtx, req.log + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.explodeArchive") + defer span.End() + utils.SetContextToRequest(ctx, c) + // remove user attributes which vary for each file in archive // to guarantee that they won't appear twice delete(filtered, object.AttributeFileName) From bfe24a458b429a38b98b77c2b08a059d3b1e7246 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 05:48:35 +0300 Subject: [PATCH 60/86] [#145] frostfs: Add spans to detail the trace Signed-off-by: Roman Loginov --- internal/service/frostfs/frostfs.go | 28 +++++++++++++++++++ .../service/frostfs/multi_object_reader.go | 7 +++++ internal/service/frostfs/tree_pool_wrapper.go | 7 +++++ 3 files changed, 42 insertions(+) diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index c6af526..4cf45a4 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -35,6 +36,9 @@ func NewFrostFS(p *pool.Pool) *FrostFS { // Container implements frostfs.FrostFS interface method. func (x *FrostFS) Container(ctx context.Context, containerPrm handler.PrmContainer) (*container.Container, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.Container") + defer span.End() + prm := pool.PrmContainerGet{ ContainerID: containerPrm.ContainerID, } @@ -49,6 +53,9 @@ func (x *FrostFS) Container(ctx context.Context, containerPrm handler.PrmContain // CreateObject implements frostfs.FrostFS interface method. func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) (oid.ID, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.CreateObject") + defer span.End() + var prmPut pool.PrmObjectPut prmPut.SetHeader(*prm.Object) prmPut.SetPayload(prm.Payload) @@ -83,6 +90,9 @@ func (x payloadReader) Read(p []byte) (int, error) { // HeadObject implements frostfs.FrostFS interface method. func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*object.Object, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.HeadObject") + defer span.End() + var prmHead pool.PrmObjectHead prmHead.SetAddress(prm.Address) @@ -100,6 +110,9 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o // GetObject implements frostfs.FrostFS interface method. func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*handler.Object, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetObject") + defer span.End() + var prmGet pool.PrmObjectGet prmGet.SetAddress(prm.Address) @@ -120,6 +133,9 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han // RangeObject implements frostfs.FrostFS interface method. func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) (io.ReadCloser, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.RangeObject") + defer span.End() + var prmRange pool.PrmObjectRange prmRange.SetAddress(prm.Address) prmRange.SetOffset(prm.PayloadRange[0]) @@ -139,6 +155,9 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) ( // SearchObjects implements frostfs.FrostFS interface method. func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch) (handler.ResObjectSearch, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.SearchObjects") + defer span.End() + var prmSearch pool.PrmObjectSearch prmSearch.SetContainerID(prm.Container) prmSearch.SetFilters(prm.Filters) @@ -157,6 +176,9 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch // GetEpochDurations implements frostfs.FrostFS interface method. func (x *FrostFS) GetEpochDurations(ctx context.Context) (*utils.EpochDurations, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetEpochDurations") + defer span.End() + networkInfo, err := x.pool.NetworkInfo(ctx) if err != nil { return nil, err @@ -175,6 +197,9 @@ func (x *FrostFS) GetEpochDurations(ctx context.Context) (*utils.EpochDurations, } func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.NetmapSnapshot") + defer span.End() + netmapSnapshot, err := x.pool.NetMapSnapshot(ctx) if err != nil { return netmapSnapshot, handleObjectError("get netmap via connection pool", err) @@ -196,6 +221,9 @@ func NewResolverFrostFS(p *pool.Pool) *ResolverFrostFS { // SystemDNS implements resolver.FrostFS interface method. func (x *ResolverFrostFS) SystemDNS(ctx context.Context) (string, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.SystemDNS") + defer span.End() + networkInfo, err := x.pool.NetworkInfo(ctx) if err != nil { return "", handleObjectError("read network info via client", err) diff --git a/internal/service/frostfs/multi_object_reader.go b/internal/service/frostfs/multi_object_reader.go index 93f1f60..b943474 100644 --- a/internal/service/frostfs/multi_object_reader.go +++ b/internal/service/frostfs/multi_object_reader.go @@ -9,6 +9,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -74,6 +75,9 @@ var ( ) func (x *FrostFS) InitMultiObjectReader(ctx context.Context, p handler.PrmInitMultiObjectReader) (io.Reader, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.InitMultiObjectReader") + defer span.End() + combinedObj, err := x.GetObject(ctx, handler.PrmObjectGet{ PrmAuth: handler.PrmAuth{BearerToken: p.Bearer}, Address: p.Addr, @@ -215,6 +219,9 @@ func (x *MultiObjectReader) Read(p []byte) (n int, err error) { // InitFrostFSObjectPayloadReader initializes payload reader of the FrostFS object. // Zero range corresponds to full payload (panics if only offset is set). func (x *FrostFS) InitFrostFSObjectPayloadReader(ctx context.Context, p GetFrostFSParams) (io.ReadCloser, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.InitFrostFSObjectPayloadReader") + defer span.End() + var prmAuth handler.PrmAuth if p.Off+p.Ln != 0 { diff --git a/internal/service/frostfs/tree_pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go index f6be05f..410acda 100644 --- a/internal/service/frostfs/tree_pool_wrapper.go +++ b/internal/service/frostfs/tree_pool_wrapper.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apitree "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/tree" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" ) @@ -46,6 +47,9 @@ func NewPoolWrapper(p *treepool.Pool) *PoolWrapper { } func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]tree.NodeResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetNodes") + defer span.End() + poolPrm := treepool.GetNodesParams{ CID: prm.CnrID, TreeID: prm.TreeID, @@ -93,6 +97,9 @@ func handleError(err error) error { } func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetSubTree") + defer span.End() + order := treepool.NoneOrder if sort { order = treepool.AscendingOrder From 412886c24fc002e6287d8ff659ecac5f749835bb Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 05:48:46 +0300 Subject: [PATCH 61/86] [#145] tree: Add spans to detail the trace Signed-off-by: Roman Loginov --- tree/tree.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tree/tree.go b/tree/tree.go index bf0aff9..315e5ad 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -190,6 +191,9 @@ func (m *multiSystemNode) Old() []*treeNode { } func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetLatestVersion") + defer span.End() + nodes, err := c.GetVersions(ctx, cnrID, objectName) if err != nil { return nil, err @@ -204,6 +208,9 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s } func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetVersions") + defer span.End() + meta := []string{oidKV, isDeleteMarkerKV, sizeKV} path := pathFromName(objectName) @@ -220,6 +227,9 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string } func (c *Tree) CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.CheckSettingsNodeExists") + defer span.End() + _, err := c.getSystemNode(ctx, bktInfo, settingsFileName) if err != nil { return err @@ -308,6 +318,9 @@ func pathFromName(objectName string) []string { } func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix") + defer span.End() + rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) if err != nil { return nil, "", err From 20319418cc33a68a3698deab00b69ef1a13d7ff5 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 11:21:53 +0300 Subject: [PATCH 62/86] [#145] Update frostfs-observability version The new version of frostfs-observability has improved the detail of tracing low-level rpc calls by adding send and receive events. Signed-off-by: Roman Loginov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d0bfcbb..b3f590e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.22 require ( - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 diff --git a/go.sum b/go.sum index 8eb4ea9..9ee2d9c 100644 --- a/go.sum +++ b/go.sum @@ -42,10 +42,10 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 h1:/Z8DfbLZXp7exUQWUKoG/9tbFdI9d5lV1qSReaYoG8I= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 47d74a5a77d5fa89955f2345c3b417c72aee1791 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Tue, 4 Feb 2025 13:21:57 +0300 Subject: [PATCH 63/86] [#174] Add slash clipping for FileName attribute According to the FrostFS API specification, the FileName attribute cannot contain a slash at the beginning. Signed-off-by: Roman Loginov --- CHANGELOG.md | 1 + internal/handler/handler.go | 14 +++++++++++++- internal/handler/handler_test.go | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30fcf7a..51080bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This document outlines major changes between releases. ### Added - Add handling quota limit reached error (#187) +- Add slash clipping for FileName attribute (#174) ## [0.32.2] - 2025-02-03 diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 69aecbf..532cdc4 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -253,6 +253,10 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte return } + if key == attrFileName { + val = prepareFileName(val) + } + log = log.With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val)) bktInfo, err := h.getBucketInfo(ctx, cidParam, log) @@ -294,7 +298,7 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cn switch { case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) - return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) + return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, prepareFileName(attrVal)) case errors.Is(err, io.EOF): log.Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) @@ -315,6 +319,14 @@ func (h *Handler) needSearchByFileName(key, val string) bool { return strings.HasPrefix(val, "/") && strings.Count(val, "/") == 1 || !strings.Contains(val, "/") } +func prepareFileName(fileName string) string { + if strings.HasPrefix(fileName, "/") { + return fileName[1:] + } + + return fileName +} + // resolveContainer decode container id, if it's not a valid container id // then trey to resolve name using provided resolver. func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 53c9739..1638f9f 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -239,6 +239,10 @@ func TestBasic(t *testing.T) { r = prepareGetByAttributeRequest(ctx, bktName, keyAttr, valAttr) hc.Handler().DownloadByAttribute(r) require.Equal(t, content, string(r.Response.Body())) + + r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, "/"+objFileName) + hc.Handler().DownloadByAttribute(r) + require.Equal(t, content, string(r.Response.Body())) }) t.Run("head by attribute", func(t *testing.T) { @@ -246,6 +250,11 @@ func TestBasic(t *testing.T) { hc.Handler().HeadByAttribute(r) require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + + r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, "/"+objFileName) + hc.Handler().HeadByAttribute(r) + require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) + require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) }) t.Run("zip", func(t *testing.T) { @@ -293,8 +302,8 @@ func TestFindObjectByAttribute(t *testing.T) { err = json.Unmarshal(r.Response.Body(), &putRes) require.NoError(t, err) - testAttrVal1 := "test-attr-val1" - testAttrVal2 := "test-attr-val2" + testAttrVal1 := "/folder/cat.jpg" + testAttrVal2 := "cat.jpg" testAttrVal3 := "test-attr-val3" for _, tc := range []struct { @@ -340,6 +349,14 @@ func TestFindObjectByAttribute(t *testing.T) { err: "not found", additionalSearch: true, }, + { + name: "success search by FilePath with leading slash (with additional search)", + firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), + secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), + reqAttrKey: attrFilePath, + reqAttrValue: "/cat.jpg", + additionalSearch: true, + }, } { t.Run(tc.name, func(t *testing.T) { obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] @@ -422,6 +439,17 @@ func TestNeedSearchByFileName(t *testing.T) { } } +func TestPrepareFileName(t *testing.T) { + fileName := "/cat.jpg" + expected := "cat.jpg" + actual := prepareFileName(fileName) + require.Equal(t, expected, actual) + + fileName = "cat.jpg" + actual = prepareFileName(fileName) + require.Equal(t, expected, actual) +} + func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) From 466f3a9531002cf8d93e8d0690133c4a7f07a639 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Wed, 5 Feb 2025 14:40:31 +0300 Subject: [PATCH 64/86] [#174] Port release v0.32.3 changelog Signed-off-by: Roman Loginov --- CHANGELOG.md | 8 +++++++- VERSION | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51080bb..2025b6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ This document outlines major changes between releases. - Add handling quota limit reached error (#187) - Add slash clipping for FileName attribute (#174) +## [0.32.3] - 2025-02-05 + +### Added +- Add slash clipping for FileName attribute (#174) + ## [0.32.2] - 2025-02-03 ### Fixed @@ -200,4 +205,5 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...v0.32.0 [0.32.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.0...v0.32.1 [0.32.2]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.1...v0.32.2 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.2...master \ No newline at end of file +[0.32.3]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.2...v0.32.3 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.3...master \ No newline at end of file diff --git a/VERSION b/VERSION index c6a2605..2c768c5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.32.2 +v0.32.3 From b362793e79880ca1083f34766fd56d08a55c3b13 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 11 Feb 2025 18:43:31 +0300 Subject: [PATCH 65/86] [#195] Use datapath tag in FrostFS pools logs Signed-off-by: Alex Vanin --- cmd/http-gw/settings.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 12d73d6..0a42a90 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -661,8 +661,8 @@ func (a *app) initPools(ctx context.Context) { errorThreshold = defaultPoolErrorThreshold } prm.SetErrorThreshold(errorThreshold) - prm.SetLogger(a.log) - prmTree.SetLogger(a.log) + prm.SetLogger(a.log.With(logs.TagField(logs.TagDatapath))) + prmTree.SetLogger(a.log.With(logs.TagField(logs.TagDatapath))) prmTree.SetMaxRequestAttempts(a.config().GetInt(cfgTreePoolMaxAttempts)) From 8bfaa841243e7d7dc22898dc45825e068b15bb08 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Tue, 18 Feb 2025 12:53:04 +0300 Subject: [PATCH 66/86] [#216] Remove http2 forcing fasthttp doesn't support http2 which causes errors when we enable it Signed-off-by: Nikita Zinkevich --- cmd/http-gw/server.go | 1 - go.mod | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/http-gw/server.go b/cmd/http-gw/server.go index 694e9ee..f8a20d9 100644 --- a/cmd/http-gw/server.go +++ b/cmd/http-gw/server.go @@ -74,7 +74,6 @@ func newServer(ctx context.Context, serverInfo ServerInfo) (*server, error) { ln = tls.NewListener(ln, &tls.Config{ GetCertificate: tlsProvider.GetCertificate, - NextProtos: []string{"h2"}, // required to enable HTTP/2 requests in `http.Serve` }) } diff --git a/go.mod b/go.mod index b3f590e..275ab52 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/net v0.30.0 golang.org/x/sys v0.28.0 google.golang.org/grpc v1.69.2 ) @@ -125,6 +124,7 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect From f9c5dc52604f42c800e7e8d9ec536f60e822dd45 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Tue, 18 Feb 2025 13:24:20 +0300 Subject: [PATCH 67/86] [#216] Rework http2 test to be tls test Signed-off-by: Nikita Zinkevich --- cmd/http-gw/server_test.go | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/cmd/http-gw/server_test.go b/cmd/http-gw/server_test.go index a937366..6f92f17 100644 --- a/cmd/http-gw/server_test.go +++ b/cmd/http-gw/server_test.go @@ -18,7 +18,7 @@ import ( "time" "github.com/stretchr/testify/require" - "golang.org/x/net/http2" + "github.com/valyala/fasthttp" ) const ( @@ -26,14 +26,10 @@ const ( expHeaderValue = "Bar" ) -func TestHTTP2TLS(t *testing.T) { +func TestHTTP_TLS(t *testing.T) { ctx := context.Background() certPath, keyPath := prepareTestCerts(t) - srv := &http.Server{ - Handler: http.HandlerFunc(testHandler), - } - tlsListener, err := newServer(ctx, ServerInfo{ Address: ":0", TLS: ServerTLSInfo{ @@ -47,37 +43,34 @@ func TestHTTP2TLS(t *testing.T) { addr := fmt.Sprintf("https://localhost:%d", port) go func() { - _ = srv.Serve(tlsListener.Listener()) + _ = fasthttp.Serve(tlsListener.Listener(), testHandler) }() - // Server is running, now send HTTP/2 request - tlsClientConfig := &tls.Config{ InsecureSkipVerify: true, } - cliHTTP1 := http.Client{Transport: &http.Transport{TLSClientConfig: tlsClientConfig}} - cliHTTP2 := http.Client{Transport: &http2.Transport{TLSClientConfig: tlsClientConfig}} + cliHTTP := http.Client{Transport: &http.Transport{}} + cliHTTPS := http.Client{Transport: &http.Transport{TLSClientConfig: tlsClientConfig}} req, err := http.NewRequest("GET", addr, nil) require.NoError(t, err) req.Header[expHeaderKey] = []string{expHeaderValue} - resp, err := cliHTTP1.Do(req) + resp, err := cliHTTPS.Do(req) require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode) - resp, err = cliHTTP2.Do(req) - require.NoError(t, err) - require.Equal(t, http.StatusOK, resp.StatusCode) + _, err = cliHTTP.Do(req) + require.ErrorContains(t, err, "failed to verify certificate") } -func testHandler(resp http.ResponseWriter, req *http.Request) { - hdr, ok := req.Header[expHeaderKey] - if !ok || len(hdr) != 1 || hdr[0] != expHeaderValue { - resp.WriteHeader(http.StatusBadRequest) +func testHandler(ctx *fasthttp.RequestCtx) { + hdr := ctx.Request.Header.Peek(expHeaderKey) + if len(hdr) == 0 || string(hdr) != expHeaderValue { + ctx.Response.SetStatusCode(http.StatusBadRequest) } else { - resp.WriteHeader(http.StatusOK) + ctx.Response.SetStatusCode(http.StatusOK) } } From a651b5823f7f3b445315c680ad54942cac0c4b0e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 21 Feb 2025 16:11:01 +0300 Subject: [PATCH 68/86] [#219] Use zaptest.Logger Use zaptest to get logs which get printed only if a test fails or if you ran go test -v. Dont use zaptest.Logger for fuzz otherwise ngfuzz/libfuzz crashes Signed-off-by: Denis Kirillov --- internal/handler/handler_fuzz_test.go | 3 ++- internal/handler/handler_test.go | 21 ++++++++++----------- internal/handler/multipart_test.go | 14 ++------------ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/internal/handler/handler_fuzz_test.go b/internal/handler/handler_fuzz_test.go index d71e8b0..ff38b11 100644 --- a/internal/handler/handler_fuzz_test.go +++ b/internal/handler/handler_fuzz_test.go @@ -21,6 +21,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" go_fuzz_utils "github.com/trailofbits/go-fuzz-utils" "github.com/valyala/fasthttp" + "go.uber.org/zap" ) const ( @@ -125,7 +126,7 @@ func maybeFillRandom(tp *go_fuzz_utils.TypeProvider, initValue string) (string, } func upload(tp *go_fuzz_utils.TypeProvider) (context.Context, *handlerContext, cid.ID, *fasthttp.RequestCtx, string, string, string, error) { - hc, err := prepareHandlerContext() + hc, err := prepareHandlerContextBase(zap.NewExample()) if err != nil { return nil, nil, cid.ID{}, nil, "", "", "", err } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 1638f9f..383dcd9 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) type treeServiceMock struct { @@ -112,12 +113,13 @@ func (hc *handlerContext) Handler() *Handler { return hc.h } -func prepareHandlerContext() (*handlerContext, error) { - logger, err := zap.NewDevelopment() - if err != nil { - return nil, err - } +func prepareHandlerContext(t *testing.T) *handlerContext { + hc, err := prepareHandlerContextBase(zaptest.NewLogger(t)) + require.NoError(t, err) + return hc +} +func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { key, err := keys.NewPrivateKey() if err != nil { return nil, err @@ -196,8 +198,7 @@ func (hc *handlerContext) prepareContainer(name string, basicACL acl.Basic) (cid } func TestBasic(t *testing.T) { - hc, err := prepareHandlerContext() - require.NoError(t, err) + hc := prepareHandlerContext(t) bktName := "bucket" cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) @@ -279,8 +280,7 @@ func TestBasic(t *testing.T) { } func TestFindObjectByAttribute(t *testing.T) { - hc, err := prepareHandlerContext() - require.NoError(t, err) + hc := prepareHandlerContext(t) hc.cfg.additionalSearch = true bktName := "bucket" @@ -377,8 +377,7 @@ func TestFindObjectByAttribute(t *testing.T) { } func TestNeedSearchByFileName(t *testing.T) { - hc, err := prepareHandlerContext() - require.NoError(t, err) + hc := prepareHandlerContext(t) for _, tc := range []struct { name string diff --git a/internal/handler/multipart_test.go b/internal/handler/multipart_test.go index 431d0d6..d7f52f4 100644 --- a/internal/handler/multipart_test.go +++ b/internal/handler/multipart_test.go @@ -60,12 +60,7 @@ func BenchmarkAll(b *testing.B) { func defaultMultipart(filename string) error { r, bound := multipartFile(filename) - logger, err := zap.NewProduction() - if err != nil { - return err - } - - file, err := fetchMultipartFileDefault(logger, r, bound) + file, err := fetchMultipartFileDefault(zap.NewNop(), r, bound) if err != nil { return err } @@ -87,12 +82,7 @@ func TestName(t *testing.T) { func customMultipart(filename string) error { r, bound := multipartFile(filename) - logger, err := zap.NewProduction() - if err != nil { - return err - } - - file, err := fetchMultipartFile(logger, r, bound) + file, err := fetchMultipartFile(zap.NewNop(), r, bound) if err != nil { return err } From cc6055bd27ec7f71d3123f4d61874eb1e41ac336 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Wed, 12 Feb 2025 11:08:59 +0300 Subject: [PATCH 69/86] [#211] Add IO tags Signed-off-by: Marina Biryukova --- cmd/http-gw/settings.go | 3 +++ go.mod | 1 + go.sum | 6 ++++-- internal/service/frostfs/frostfs.go | 13 ++++++++----- internal/service/frostfs/tree_pool_wrapper.go | 5 +++-- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 0a42a90..69ecce2 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -21,6 +21,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "github.com/spf13/pflag" @@ -670,6 +671,8 @@ func (a *app) initPools(ctx context.Context) { grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), grpc.WithContextDialer(a.settings.dialerSource.GrpcContextDialer()), + grpc.WithChainUnaryInterceptor(qostagging.NewUnaryClientInteceptor()), + grpc.WithChainStreamInterceptor(qostagging.NewStreamClientInterceptor()), } prm.SetGRPCDialOptions(interceptors...) prmTree.SetGRPCDialOptions(interceptors...) diff --git a/go.mod b/go.mod index 275ab52..0ace5f2 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 + git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 diff --git a/go.sum b/go.sum index 9ee2d9c..a2121ab 100644 --- a/go.sum +++ b/go.sum @@ -42,10 +42,12 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 h1:/Z8DfbLZXp7exUQWUKoG/9tbFdI9d5lV1qSReaYoG8I= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe h1:81gDNdWNLP24oMQukRiCE9R1wGSh0l0dRq3F1W+Oesc= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index 4cf45a4..9115930 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -20,6 +21,8 @@ import ( "google.golang.org/grpc/status" ) +const clientIOTag = "client" + // FrostFS represents virtual connection to the FrostFS network. // It is used to provide an interface to dependent packages // which work with FrostFS. @@ -67,7 +70,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) prmPut.UseBearer(*prm.BearerToken) } - idObj, err := x.pool.PutObject(ctx, prmPut) + idObj, err := x.pool.PutObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmPut) if err != nil { return oid.ID{}, handleObjectError("save object via connection pool", err) } @@ -100,7 +103,7 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o prmHead.UseBearer(*prm.BearerToken) } - res, err := x.pool.HeadObject(ctx, prmHead) + res, err := x.pool.HeadObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmHead) if err != nil { return nil, handleObjectError("read object header via connection pool", err) } @@ -120,7 +123,7 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han prmGet.UseBearer(*prm.BearerToken) } - res, err := x.pool.GetObject(ctx, prmGet) + res, err := x.pool.GetObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmGet) if err != nil { return nil, handleObjectError("init full object reading via connection pool", err) } @@ -145,7 +148,7 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) ( prmRange.UseBearer(*prm.BearerToken) } - res, err := x.pool.ObjectRange(ctx, prmRange) + res, err := x.pool.ObjectRange(qostagging.ContextWithIOTag(ctx, clientIOTag), prmRange) if err != nil { return nil, handleObjectError("init payload range reading via connection pool", err) } @@ -166,7 +169,7 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch prmSearch.UseBearer(*prm.BearerToken) } - res, err := x.pool.SearchObjects(ctx, prmSearch) + res, err := x.pool.SearchObjects(qostagging.ContextWithIOTag(ctx, clientIOTag), prmSearch) if err != nil { return nil, handleObjectError("init object search via connection pool", err) } diff --git a/internal/service/frostfs/tree_pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go index 410acda..89afc3c 100644 --- a/internal/service/frostfs/tree_pool_wrapper.go +++ b/internal/service/frostfs/tree_pool_wrapper.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apitree "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/tree" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" ) @@ -61,7 +62,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([ BearerToken: getBearer(ctx), } - nodes, err := w.p.GetNodes(ctx, poolPrm) + nodes, err := w.p.GetNodes(qostagging.ContextWithIOTag(ctx, clientIOTag), poolPrm) if err != nil { return nil, handleError(err) } @@ -120,7 +121,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, poolPrm.RootID = nil } - subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) + subTreeReader, err := w.p.GetSubTree(qostagging.ContextWithIOTag(ctx, clientIOTag), poolPrm) if err != nil { return nil, handleError(err) } From 9cf2a4f0e0011bf0ae87482a36b2055405c736e8 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Sun, 9 Feb 2025 21:48:32 +0300 Subject: [PATCH 70/86] [#197] Add a leading slash to the FilePath attribute According to the frostfs api specification, the File Path attribute must start with a leading slash. More info: https://git.frostfs.info/TrueCloudLab/frostfs-api Signed-off-by: Roman Loginov --- internal/handler/handler.go | 27 ++++++++++++++++++++++---- internal/handler/handler_test.go | 33 +++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 532cdc4..179cf60 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -253,9 +253,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte return } - if key == attrFileName { - val = prepareFileName(val) - } + val = prepareAtribute(key, val) log = log.With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val)) @@ -298,7 +296,8 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cn switch { case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) - return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, prepareFileName(attrVal)) + attrVal = prepareAtribute(attrFileName, attrVal) + return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) case errors.Is(err, io.EOF): log.Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) @@ -319,6 +318,18 @@ func (h *Handler) needSearchByFileName(key, val string) bool { return strings.HasPrefix(val, "/") && strings.Count(val, "/") == 1 || !strings.Contains(val, "/") } +func prepareAtribute(attrKey, attrVal string) string { + if attrKey == attrFileName { + return prepareFileName(attrVal) + } + + if attrKey == attrFilePath { + return prepareFilePath(attrVal) + } + + return attrVal +} + func prepareFileName(fileName string) string { if strings.HasPrefix(fileName, "/") { return fileName[1:] @@ -327,6 +338,14 @@ func prepareFileName(fileName string) string { return fileName } +func prepareFilePath(filePath string) string { + if !strings.HasPrefix(filePath, "/") { + return "/" + filePath + } + + return filePath +} + // resolveContainer decode container id, if it's not a valid container id // then trey to resolve name using provided resolver. func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 383dcd9..ab2cd9f 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -220,8 +220,10 @@ func TestBasic(t *testing.T) { require.NoError(t, err) obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] - attr := prepareObjectAttributes(object.AttributeFilePath, objFileName) - obj.SetAttributes(append(obj.Attributes(), attr)...) + fileName := prepareObjectAttributes(object.AttributeFileName, objFileName) + filePath := prepareObjectAttributes(object.AttributeFilePath, objFilePath) + obj.SetAttributes(append(obj.Attributes(), fileName)...) + obj.SetAttributes(append(obj.Attributes(), filePath)...) t.Run("get", func(t *testing.T) { r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID) @@ -241,7 +243,11 @@ func TestBasic(t *testing.T) { hc.Handler().DownloadByAttribute(r) require.Equal(t, content, string(r.Response.Body())) - r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, "/"+objFileName) + r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath) + hc.Handler().DownloadByAttribute(r) + require.Equal(t, content, string(r.Response.Body())) + + r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName) hc.Handler().DownloadByAttribute(r) require.Equal(t, content, string(r.Response.Body())) }) @@ -252,7 +258,12 @@ func TestBasic(t *testing.T) { require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) - r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, "/"+objFileName) + r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath) + hc.Handler().HeadByAttribute(r) + require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) + require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + + r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName) hc.Handler().HeadByAttribute(r) require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) @@ -266,7 +277,7 @@ func TestBasic(t *testing.T) { zipReader, err := zip.NewReader(readerAt, int64(len(r.Response.Body()))) require.NoError(t, err) require.Len(t, zipReader.File, 1) - require.Equal(t, objFileName, zipReader.File[0].Name) + require.Equal(t, objFilePath, zipReader.File[0].Name) f, err := zipReader.File[0].Open() require.NoError(t, err) defer func() { @@ -449,6 +460,17 @@ func TestPrepareFileName(t *testing.T) { require.Equal(t, expected, actual) } +func TestPrepareFilePath(t *testing.T) { + filePath := "cat.jpg" + expected := "/cat.jpg" + actual := prepareFilePath(filePath) + require.Equal(t, expected, actual) + + filePath = "/cat.jpg" + actual = prepareFilePath(filePath) + require.Equal(t, expected, actual) +} + func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) @@ -492,6 +514,7 @@ const ( keyAttr = "User-Attribute" valAttr = "user value" objFileName = "newFile.txt" + objFilePath = "/newFile.txt" ) func fillMultipartBody(r *fasthttp.RequestCtx, content string) error { From 9ef6b06e91899aa4f2f3c32dd0179f3cf9282c2f Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Thu, 27 Feb 2025 12:13:12 +0300 Subject: [PATCH 71/86] [#212] Support CORS container for CORS settings Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 206 ++++---- cmd/http-gw/integration_test.go | 21 +- cmd/http-gw/settings.go | 43 +- config/config.env | 6 + config/config.yaml | 7 + docs/gate-configuration.md | 20 +- internal/cache/cors.go | 62 +++ internal/data/cors.go | 18 + internal/handler/cors.go | 342 ++++++++++++++ internal/handler/cors_test.go | 440 ++++++++++++++++++ internal/handler/frostfs_mock.go | 4 + internal/handler/handler.go | 29 +- internal/handler/handler_test.go | 60 ++- internal/logs/logs.go | 68 +-- internal/service/frostfs/frostfs.go | 13 +- internal/service/frostfs/tree_pool_wrapper.go | 5 +- resolver/resolver.go | 52 +-- utils/attributes.go | 11 + 18 files changed, 1204 insertions(+), 203 deletions(-) create mode 100644 internal/cache/cors.go create mode 100644 internal/data/cors.go create mode 100644 internal/handler/cors.go create mode 100644 internal/handler/cors_test.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 103c72b..c75f9d8 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -17,6 +17,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" @@ -30,6 +31,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -65,6 +67,8 @@ type ( settings *appSettings loggerSettings *loggerSettings bucketCache *cache.BucketCache + handle *handler.Handler + corsCnrID cid.ID servers []Server unbindServers []ServerInfo @@ -105,12 +109,7 @@ type ( bufferMaxSizeForPut uint64 namespaceHeader string defaultNamespaces []string - corsAllowOrigin string - corsAllowMethods []string - corsAllowHeaders []string - corsExposeHeaders []string - corsAllowCredentials bool - corsMaxAge int + cors *data.CORSRule enableFilepathFallback bool } @@ -122,15 +121,6 @@ type ( logLevel zap.AtomicLevel tagsConfig *tagsConfig } - - CORS struct { - AllowOrigin string - AllowMethods []string - AllowHeaders []string - ExposeHeaders []string - AllowCredentials bool - MaxAge int - } ) func newLogLevel(v *viper.Viper) zap.AtomicLevel { @@ -251,6 +241,7 @@ func newApp(ctx context.Context, cfg *appCfg) App { a.initResolver() a.initMetrics() a.initTracing(ctx) + a.initContainers(ctx) return a } @@ -259,6 +250,14 @@ func (a *app) config() *viper.Viper { return a.cfg.config() } +func (a *app) initContainers(ctx context.Context) { + corsCnrID, err := a.fetchContainerID(ctx, cfgContainersCORS) + if err != nil { + a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err), logs.TagField(logs.TagApp)) + } + a.corsCnrID = *corsCnrID +} + func (a *app) initAppSettings(lc *logLevelConfig) { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.config()), @@ -278,12 +277,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { namespaceHeader := v.GetString(cfgResolveNamespaceHeader) defaultNamespaces := fetchDefaultNamespaces(v) indexPage, indexEnabled := fetchIndexPageTemplate(v, l) - corsAllowOrigin := v.GetString(cfgCORSAllowOrigin) - corsAllowMethods := v.GetStringSlice(cfgCORSAllowMethods) - corsAllowHeaders := v.GetStringSlice(cfgCORSAllowHeaders) - corsExposeHeaders := v.GetStringSlice(cfgCORSExposeHeaders) - corsAllowCredentials := v.GetBool(cfgCORSAllowCredentials) - corsMaxAge := fetchCORSMaxAge(v) + cors := fetchCORSConfig(v) enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback) s.mu.Lock() @@ -298,12 +292,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.defaultNamespaces = defaultNamespaces s.returnIndexPage = indexEnabled s.indexPageTemplate = indexPage - s.corsAllowOrigin = corsAllowOrigin - s.corsAllowMethods = corsAllowMethods - s.corsAllowHeaders = corsAllowHeaders - s.corsExposeHeaders = corsExposeHeaders - s.corsAllowCredentials = corsAllowCredentials - s.corsMaxAge = corsMaxAge + s.cors = cors s.enableFilepathFallback = enableFilepathFallback } @@ -350,26 +339,33 @@ func (s *appSettings) IndexPageTemplate() string { return s.indexPageTemplate } -func (s *appSettings) CORS() CORS { +func (s *appSettings) CORS() *data.CORSRule { s.mu.RLock() defer s.mu.RUnlock() - allowMethods := make([]string, len(s.corsAllowMethods)) - copy(allowMethods, s.corsAllowMethods) + if s.cors == nil { + return nil + } - allowHeaders := make([]string, len(s.corsAllowHeaders)) - copy(allowHeaders, s.corsAllowHeaders) + allowMethods := make([]string, len(s.cors.AllowedMethods)) + copy(allowMethods, s.cors.AllowedMethods) - exposeHeaders := make([]string, len(s.corsExposeHeaders)) - copy(exposeHeaders, s.corsExposeHeaders) + allowHeaders := make([]string, len(s.cors.AllowedHeaders)) + copy(allowHeaders, s.cors.AllowedHeaders) - return CORS{ - AllowOrigin: s.corsAllowOrigin, - AllowMethods: allowMethods, - AllowHeaders: allowHeaders, - ExposeHeaders: exposeHeaders, - AllowCredentials: s.corsAllowCredentials, - MaxAge: s.corsMaxAge, + exposeHeaders := make([]string, len(s.cors.ExposeHeaders)) + copy(exposeHeaders, s.cors.ExposeHeaders) + + allowOrigins := make([]string, len(s.cors.AllowedOrigins)) + copy(allowOrigins, s.cors.AllowedOrigins) + + return &data.CORSRule{ + AllowedOrigins: allowOrigins, + AllowedMethods: allowMethods, + AllowedHeaders: allowHeaders, + ExposeHeaders: exposeHeaders, + AllowedCredentials: s.cors.AllowedCredentials, + MaxAgeSeconds: s.cors.MaxAgeSeconds, } } @@ -391,15 +387,15 @@ func (s *appSettings) NamespaceHeader() string { return s.namespaceHeader } -func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) { +func (s *appSettings) FormContainerZone(ns string) string { s.mu.RLock() namespaces := s.defaultNamespaces s.mu.RUnlock() if slices.Contains(namespaces, ns) { - return v2container.SysAttributeZoneDefault, true + return v2container.SysAttributeZoneDefault } - return ns + ".ns", false + return ns + ".ns" } func (s *appSettings) EnableFilepathFallback() bool { @@ -420,7 +416,6 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) { resolveCfg := &resolver.Config{ FrostFS: frostfs.NewResolverFrostFS(a.pool), RPCAddress: a.config().GetString(cfgRPCEndpoint), - Settings: a.settings, } order := a.config().GetStringSlice(cfgResolveOrder) @@ -606,10 +601,8 @@ func (a *app) Serve() { close(a.webDone) }() - handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) - // Configure router. - a.configureRouter(handle) + a.configureRouter(workerPool) a.startServices() a.initServers(a.ctx) @@ -730,7 +723,9 @@ func (a *app) stopServices() { } } -func (a *app) configureRouter(h *handler.Handler) { +func (a *app) configureRouter(workerPool *ants.Pool) { + a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) + r := router.New() r.RedirectTrailingSlash = true r.NotFound = func(r *fasthttp.RequestCtx) { @@ -740,21 +735,21 @@ func (a *app) configureRouter(h *handler.Handler) { handler.ResponseError(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.addMiddlewares(h.Upload)) - r.OPTIONS("/upload/{cid}", a.addPreflight()) + r.POST("/upload/{cid}", a.addMiddlewares(a.handle.Upload)) + r.OPTIONS("/upload/{cid}", a.addPreflight(a.handle.Preflight)) a.log.Info(logs.AddedPathUploadCid, logs.TagField(logs.TagApp)) - r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(h.DownloadByAddressOrBucketName)) - r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(h.HeadByAddressOrBucketName)) - r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight()) + r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(a.handle.DownloadByAddressOrBucketName)) + r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(a.handle.HeadByAddressOrBucketName)) + r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight(a.handle.Preflight)) a.log.Info(logs.AddedPathGetCidOid, logs.TagField(logs.TagApp)) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.DownloadByAttribute)) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute)) - r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(a.handle.DownloadByAttribute)) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(a.handle.HeadByAttribute)) + r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight(a.handle.Preflight)) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal, logs.TagField(logs.TagApp)) - r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZip)) - r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) - r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadTar)) - r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight()) + r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(a.handle.DownloadZip)) + r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight(a.handle.Preflight)) + r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(a.handle.DownloadTar)) + r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight(a.handle.Preflight)) a.log.Info(logs.AddedPathZipCidPrefix, logs.TagField(logs.TagApp)) a.webServer.Handler = r.Handler @@ -777,14 +772,14 @@ func (a *app) addMiddlewares(h fasthttp.RequestHandler) fasthttp.RequestHandler return h } -func (a *app) addPreflight() fasthttp.RequestHandler { +func (a *app) addPreflight(h fasthttp.RequestHandler) fasthttp.RequestHandler { list := []func(fasthttp.RequestHandler) fasthttp.RequestHandler{ a.tracer, a.logger, + a.canonicalizer, a.reqNamespace, } - h := a.preflightHandler for i := len(list) - 1; i >= 0; i-- { h = list[i](h) } @@ -792,46 +787,16 @@ func (a *app) addPreflight() fasthttp.RequestHandler { return h } -func (a *app) preflightHandler(c *fasthttp.RequestCtx) { - cors := a.settings.CORS() - setCORSHeaders(c, cors) -} - func (a *app) cors(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(c *fasthttp.RequestCtx) { h(c) code := c.Response.StatusCode() if code >= fasthttp.StatusOK && code < fasthttp.StatusMultipleChoices { - cors := a.settings.CORS() - setCORSHeaders(c, cors) + a.handle.SetCORSHeaders(c) } } } -func setCORSHeaders(c *fasthttp.RequestCtx, cors CORS) { - c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(cors.MaxAge)) - - if len(cors.AllowOrigin) != 0 { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, cors.AllowOrigin) - } - - if len(cors.AllowMethods) != 0 { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(cors.AllowMethods, ",")) - } - - if len(cors.AllowHeaders) != 0 { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, strings.Join(cors.AllowHeaders, ",")) - } - - if len(cors.ExposeHeaders) != 0 { - c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(cors.ExposeHeaders, ",")) - } - - if cors.AllowCredentials { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") - } -} - func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { requiredFields := []zap.Field{zap.Uint64("id", req.ID())} @@ -930,11 +895,13 @@ func (a *app) reqNamespace(h fasthttp.RequestHandler) fasthttp.RequestHandler { func (a *app) AppParams() *handler.AppParams { return &handler.AppParams{ - Logger: a.log, - FrostFS: frostfs.NewFrostFS(a.pool), - Owner: a.owner, - Resolver: a.resolver, - Cache: a.bucketCache, + Logger: a.log, + FrostFS: frostfs.NewFrostFS(a.pool), + Owner: a.owner, + Resolver: a.resolver, + Cache: a.bucketCache, + CORSCnrID: a.corsCnrID, + CORSCache: cache.NewCORSCache(getCORSCacheOptions(a.config(), a.log)), } } @@ -1135,3 +1102,44 @@ func (a *app) tryReconnect(ctx context.Context, sr *fasthttp.Server) bool { return len(a.unbindServers) == 0 } + +func (a *app) fetchContainerID(ctx context.Context, cfgKey string) (id *cid.ID, err error) { + cnrID, err := a.resolveContainerID(ctx, cfgKey) + if err != nil { + return nil, err + } + + err = checkContainerExists(ctx, *cnrID, a.pool) + if err != nil { + return nil, err + } + + return cnrID, nil +} + +func (a *app) resolveContainerID(ctx context.Context, cfgKey string) (*cid.ID, error) { + containerString := a.config().GetString(cfgKey) + + id := new(cid.ID) + if err := id.DecodeString(containerString); err != nil { + i := strings.Index(containerString, ".") + if i < 0 { + return nil, fmt.Errorf("invalid container address: %s", containerString) + } + + if id, err = a.resolver.Resolve(ctx, containerString[i+1:], containerString[:i]); err != nil { + return nil, fmt.Errorf("resolve container address %s: %w", containerString, err) + } + } + + return id, nil +} + +func checkContainerExists(ctx context.Context, id cid.ID, frostFSPool *pool.Pool) error { + prm := pool.PrmContainerGet{ + ContainerID: id, + } + + _, err := frostFSPool.GetContainer(ctx, prm) + return err +} diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 2596bee..20b4c8b 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -43,9 +43,10 @@ type putResponse struct { } const ( - testContainerName = "friendly" - testListenAddress = "localhost:8082" - testHost = "http://" + testListenAddress + testContainerName = "friendly" + testListenAddress = "localhost:8082" + testHost = "http://" + testListenAddress + testCORSContainerName = "cors" ) func TestIntegration(t *testing.T) { @@ -76,10 +77,14 @@ func TestIntegration(t *testing.T) { registerUser(t, ctx, aioContainer, file.Name()) } + // Creating CORS container + clientPool := getPool(ctx, t, key) + _, err = createContainer(ctx, t, clientPool, ownerID, testCORSContainerName) + require.NoError(t, err, version) + // See the logs from the command execution. server, cancel := runServer(file.Name()) - clientPool := getPool(ctx, t, key) - CID, err := createContainer(ctx, t, clientPool, ownerID) + CID, err := createContainer(ctx, t, clientPool, ownerID, testContainerName) require.NoError(t, err, version) jsonToken, binaryToken := makeBearerTokens(t, key, ownerID, version) @@ -110,6 +115,8 @@ func runServer(pathToWallet string) (App, context.CancelFunc) { v.config().Set(cfgWalletPath, pathToWallet) v.config().Set(cfgWalletPassphrase, "") + v.config().Set(cfgContainersCORS, testCORSContainerName+"."+containerv2.SysAttributeZoneDefault) + application := newApp(cancelCtx, v) go application.Serve() @@ -477,7 +484,7 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool return clientPool } -func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID) (cid.ID, error) { +func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, name string) (cid.ID, error) { var policy netmap.PlacementPolicy err := policy.DecodeString("REP 1") require.NoError(t, err) @@ -491,7 +498,7 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o container.SetCreationTime(&cnr, time.Now()) var domain container.Domain - domain.SetName(testContainerName) + domain.SetName(name) cnr.SetAttribute(containerv2.SysAttributeName, domain.Name()) cnr.SetAttribute(containerv2.SysAttributeZone, domain.Zone()) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 69ecce2..132c627 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -16,6 +16,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" @@ -155,18 +156,21 @@ const ( cfgBucketsCacheLifetime = "cache.buckets.lifetime" cfgBucketsCacheSize = "cache.buckets.size" cfgNetmapCacheLifetime = "cache.netmap.lifetime" + cfgCORSCacheLifetime = "cache.cors.lifetime" + cfgCORSCacheSize = "cache.cors.size" // Bucket resolving options. cfgResolveNamespaceHeader = "resolve_bucket.namespace_header" cfgResolveDefaultNamespaces = "resolve_bucket.default_namespaces" // CORS. - cfgCORSAllowOrigin = "cors.allow_origin" - cfgCORSAllowMethods = "cors.allow_methods" - cfgCORSAllowHeaders = "cors.allow_headers" - cfgCORSExposeHeaders = "cors.expose_headers" - cfgCORSAllowCredentials = "cors.allow_credentials" - cfgCORSMaxAge = "cors.max_age" + cfgCORS = "cors" + cfgCORSAllowOrigin = cfgCORS + ".allow_origin" + cfgCORSAllowMethods = cfgCORS + ".allow_methods" + cfgCORSAllowHeaders = cfgCORS + ".allow_headers" + cfgCORSExposeHeaders = cfgCORS + ".expose_headers" + cfgCORSAllowCredentials = cfgCORS + ".allow_credentials" + cfgCORSMaxAge = cfgCORS + ".max_age" // Multinet. cfgMultinetEnabled = "multinet.enabled" @@ -179,6 +183,9 @@ const ( cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" + // Containers. + cfgContainersCORS = "containers.cors" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -759,6 +766,15 @@ func getNetmapCacheOptions(v *viper.Viper, l *zap.Logger) *cache.NetmapCacheConf return cacheCfg } +func getCORSCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { + cacheCfg := cache.DefaultCORSConfig(l) + + cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgCORSCacheLifetime, cacheCfg.Lifetime) + cacheCfg.Size = fetchCacheSize(v, l, cfgCORSCacheSize, cacheCfg.Size) + + return cacheCfg +} + func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration { if v.IsSet(cfgEntry) { lifetime := v.GetDuration(cfgEntry) @@ -854,3 +870,18 @@ func fetchArchiveCompression(v *viper.Viper) bool { } return v.GetBool(cfgArchiveCompression) } + +func fetchCORSConfig(v *viper.Viper) *data.CORSRule { + if !v.IsSet(cfgCORS) { + return nil + } + + return &data.CORSRule{ + AllowedOrigins: []string{v.GetString(cfgCORSAllowOrigin)}, + AllowedMethods: v.GetStringSlice(cfgCORSAllowMethods), + AllowedHeaders: v.GetStringSlice(cfgCORSAllowHeaders), + ExposeHeaders: v.GetStringSlice(cfgCORSExposeHeaders), + AllowedCredentials: v.GetBool(cfgCORSAllowCredentials), + MaxAgeSeconds: fetchCORSMaxAge(v), + } +} diff --git a/config/config.env b/config/config.env index af0eba1..0ff2dec 100644 --- a/config/config.env +++ b/config/config.env @@ -129,6 +129,9 @@ HTTP_GW_CACHE_BUCKETS_LIFETIME=1m HTTP_GW_CACHE_BUCKETS_SIZE=1000 # Cache which stores netmap HTTP_GW_CACHE_NETMAP_LIFETIME=1m +# Cache which stores container CORS configurations +HTTP_GW_CACHE_CORS_LIFETIME=5m +HTTP_GW_CACHE_CORS_SIZE=1000 # Header to determine zone to resolve bucket name HTTP_GW_RESOLVE_BUCKET_NAMESPACE_HEADER=X-Frostfs-Namespace @@ -172,3 +175,6 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true + +# Containers properties +HTTP_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj diff --git a/config/config.yaml b/config/config.yaml index 8c51591..05bba2e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -156,6 +156,10 @@ cache: # Cache which stores netmap netmap: lifetime: 1m + # Cache which stores container CORS configurations + cors: + lifetime: 5m + size: 1000 resolve_bucket: namespace_header: X-Frostfs-Namespace @@ -191,3 +195,6 @@ features: enable_filepath_fallback: false # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service tree_pool_netmap_support: true + +containers: + cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 191e9bb..628d3c7 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -60,6 +60,7 @@ $ cat http.log | `index_page` | [Index page configuration](#index_page-section) | | `multinet` | [Multinet configuration](#multinet-section) | | `features` | [Features configuration](#features-section) | +| `containers` | [Containers configuration](#containers-section) | # General section @@ -382,12 +383,16 @@ cache: size: 1000 netmap: lifetime: 1m + cors: + lifetime: 5m + size: 1000 ``` | Parameter | Type | Default value | Description | |-----------|-----------------------------------|---------------------------------|---------------------------------------------------------------------------| | `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`
`size: 1000` | Cache which contains mapping of bucket name to bucket info. | | `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. | +| `cors` | [Cache config](#cache-subsection) | `lifetime: 5m`
`size: 1000` | Cache which stores container CORS configurations. | #### `cache` subsection @@ -441,7 +446,7 @@ index_page: # `cors` section Parameters for CORS (used in OPTIONS requests and responses in all handlers). -If values are not set, headers will not be included to response. +If values are not set, settings from CORS container will be used. ```yaml cors: @@ -515,3 +520,16 @@ features: |-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. | | `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | + +# `containers` section + +Section for well-known containers to store data and settings. + +```yaml +containers: + cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-------------|----------|---------------|---------------|-----------------------------------------| +| `cors` | `string` | no | | Container name for CORS configurations. | diff --git a/internal/cache/cors.go b/internal/cache/cors.go new file mode 100644 index 0000000..24465b8 --- /dev/null +++ b/internal/cache/cors.go @@ -0,0 +1,62 @@ +package cache + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/bluele/gcache" + "go.uber.org/zap" +) + +// CORSCache contains cache with CORS objects. +type CORSCache struct { + cache gcache.Cache + logger *zap.Logger +} + +const ( + // DefaultCORSCacheSize is a default maximum number of entries in cache. + DefaultCORSCacheSize = 1e3 + // DefaultCORSCacheLifetime is a default lifetime of entries in cache. + DefaultCORSCacheLifetime = 5 * time.Minute +) + +// DefaultCORSConfig returns new default cache expiration values. +func DefaultCORSConfig(logger *zap.Logger) *Config { + return &Config{ + Size: DefaultCORSCacheSize, + Lifetime: DefaultCORSCacheLifetime, + Logger: logger, + } +} + +// NewCORSCache creates an object of CORSCache. +func NewCORSCache(config *Config) *CORSCache { + gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + return &CORSCache{cache: gc, logger: config.Logger} +} + +// Get returns a cached object. +func (o *CORSCache) Get(cnrID cid.ID) *data.CORSConfiguration { + entry, err := o.cache.Get(cnrID) + if err != nil { + return nil + } + + result, ok := entry.(*data.CORSConfiguration) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath)) + return nil + } + + return result +} + +// Put puts an object to cache. +func (o *CORSCache) Put(cnrID cid.ID, cors *data.CORSConfiguration) error { + return o.cache.Set(cnrID, cors) +} diff --git a/internal/data/cors.go b/internal/data/cors.go new file mode 100644 index 0000000..d1b1106 --- /dev/null +++ b/internal/data/cors.go @@ -0,0 +1,18 @@ +package data + +type ( + // CORSConfiguration stores CORS configuration of a request. + CORSConfiguration struct { + CORSRules []CORSRule `xml:"CORSRule" json:"CORSRules"` + } + + // CORSRule stores rules for CORS configuration. + CORSRule struct { + AllowedHeaders []string `xml:"AllowedHeader" json:"AllowedHeaders"` + AllowedMethods []string `xml:"AllowedMethod" json:"AllowedMethods"` + AllowedOrigins []string `xml:"AllowedOrigin" json:"AllowedOrigins"` + ExposeHeaders []string `xml:"ExposeHeader" json:"ExposeHeaders"` + MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty" json:"MaxAgeSeconds,omitempty"` + AllowedCredentials bool `xml:"AllowedCredentials,omitempty" json:"AllowedCredentials,omitempty"` + } +) diff --git a/internal/handler/cors.go b/internal/handler/cors.go new file mode 100644 index 0000000..234ef2a --- /dev/null +++ b/internal/handler/cors.go @@ -0,0 +1,342 @@ +package handler + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "sort" + "strconv" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +const ( + internalIOTag = "internal" + corsFilePathTemplate = "/%s.cors" + wildcard = "*" +) + +var errNoCORS = errors.New("no CORS objects found") + +func (h *Handler) Preflight(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.Preflight") + defer span.End() + + ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) + cidParam, _ := c.UserValue("cid").(string) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("cid", cidParam)) + + origin := c.Request.Header.Peek(fasthttp.HeaderOrigin) + if len(origin) == 0 { + log.Error(logs.EmptyOriginRequestHeader, logs.TagField(logs.TagDatapath)) + ResponseError(c, "Origin request header needed", fasthttp.StatusBadRequest) + return + } + + method := c.Request.Header.Peek(fasthttp.HeaderAccessControlRequestMethod) + if len(method) == 0 { + log.Error(logs.EmptyAccessControlRequestMethodHeader, logs.TagField(logs.TagDatapath)) + ResponseError(c, "Access-Control-Request-Method request header needed", fasthttp.StatusBadRequest) + return + } + + corsRule := h.config.CORS() + if corsRule != nil { + setCORSHeadersFromRule(c, corsRule) + return + } + + corsConfig, err := h.getCORSConfig(ctx, log, cidParam) + if err != nil { + log.Error(logs.CouldNotGetCORSConfiguration, zap.Error(err), logs.TagField(logs.TagDatapath)) + status := fasthttp.StatusInternalServerError + if errors.Is(err, errNoCORS) { + status = fasthttp.StatusNotFound + } + ResponseError(c, "could not get CORS configuration: "+err.Error(), status) + return + } + + var headers []string + requestHeaders := c.Request.Header.Peek(fasthttp.HeaderAccessControlRequestHeaders) + if len(requestHeaders) > 0 { + headers = strings.Split(string(requestHeaders), ", ") + } + + for _, rule := range corsConfig.CORSRules { + for _, o := range rule.AllowedOrigins { + if o == string(origin) || o == wildcard { + for _, m := range rule.AllowedMethods { + if m == string(method) { + if !checkSubslice(rule.AllowedHeaders, headers) { + continue + } + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + if headers != nil { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, string(requestHeaders)) + } + if rule.ExposeHeaders != nil { + c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(rule.ExposeHeaders, ", ")) + } + if rule.MaxAgeSeconds > 0 || rule.MaxAgeSeconds == -1 { + c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(rule.MaxAgeSeconds)) + } + if o != wildcard { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + } + return + } + } + } + } + } + log.Error(logs.CORSRuleWasNotMatched, logs.TagField(logs.TagDatapath)) + ResponseError(c, "Forbidden", fasthttp.StatusForbidden) +} + +func (h *Handler) SetCORSHeaders(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.SetCORSHeaders") + defer span.End() + + origin := c.Request.Header.Peek(fasthttp.HeaderOrigin) + if len(origin) == 0 { + return + } + + ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) + cidParam, _ := c.UserValue("cid").(string) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("cid", cidParam)) + + corsRule := h.config.CORS() + if corsRule != nil { + setCORSHeadersFromRule(c, corsRule) + return + } + + corsConfig, err := h.getCORSConfig(ctx, log, cidParam) + if err != nil { + log.Error(logs.CouldNotGetCORSConfiguration, zap.Error(err), logs.TagField(logs.TagDatapath)) + return + } + + var withCredentials bool + if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { + withCredentials = true + } + + for _, rule := range corsConfig.CORSRules { + for _, o := range rule.AllowedOrigins { + if o == string(origin) { + for _, m := range rule.AllowedMethods { + if m == string(c.Method()) { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + c.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) + return + } + } + } + if o == wildcard { + for _, m := range rule.AllowedMethods { + if m == string(c.Method()) { + if withCredentials { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + c.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) + } else { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, o) + } + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + return + } + } + } + } + } +} + +func (h *Handler) getCORSConfig(ctx context.Context, log *zap.Logger, cidStr string) (*data.CORSConfiguration, error) { + cnrID, err := h.resolveContainer(ctx, cidStr) + if err != nil { + return nil, fmt.Errorf("resolve container '%s': %w", cidStr, err) + } + + if cors := h.corsCache.Get(*cnrID); cors != nil { + return cors, nil + } + + objID, err := h.getLastCORSObject(ctx, *cnrID) + if err != nil { + return nil, fmt.Errorf("get last cors object: %w", err) + } + + var addr oid.Address + addr.SetContainer(h.corsCnrID) + addr.SetObject(objID) + corsObj, err := h.frostfs.GetObject(ctx, PrmObjectGet{ + PrmAuth: PrmAuth{ + BearerToken: bearerToken(ctx), + }, + Address: addr, + }) + if err != nil { + return nil, fmt.Errorf("get cors object '%s': %w", addr.EncodeToString(), err) + } + + corsConfig := &data.CORSConfiguration{} + if err = xml.NewDecoder(corsObj.Payload).Decode(corsConfig); err != nil { + return nil, fmt.Errorf("decode cors object: %w", err) + } + + if err = h.corsCache.Put(*cnrID, corsConfig); err != nil { + log.Warn(logs.CouldntCacheCors, zap.Error(err), logs.TagField(logs.TagDatapath)) + } + + return corsConfig, nil +} + +func (h *Handler) getLastCORSObject(ctx context.Context, cnrID cid.ID) (oid.ID, error) { + filters := object.NewSearchFilters() + filters.AddRootFilter() + filters.AddFilter(object.AttributeFilePath, fmt.Sprintf(corsFilePathTemplate, cnrID), object.MatchStringEqual) + + prmAuth := PrmAuth{ + BearerToken: bearerToken(ctx), + } + res, err := h.frostfs.SearchObjects(ctx, PrmObjectSearch{ + PrmAuth: prmAuth, + Container: h.corsCnrID, + Filters: filters, + }) + if err != nil { + return oid.ID{}, fmt.Errorf("search cors versions: %w", err) + } + defer res.Close() + + var ( + addr oid.Address + obj *object.Object + headErr error + objs = make([]*object.Object, 0) + ) + addr.SetContainer(h.corsCnrID) + err = res.Iterate(func(id oid.ID) bool { + addr.SetObject(id) + obj, headErr = h.frostfs.HeadObject(ctx, PrmObjectHead{ + PrmAuth: prmAuth, + Address: addr, + }) + if headErr != nil { + headErr = fmt.Errorf("head cors object '%s': %w", addr.EncodeToString(), headErr) + return true + } + + objs = append(objs, obj) + return false + }) + if err != nil { + return oid.ID{}, fmt.Errorf("iterate cors objects: %w", err) + } + + if headErr != nil { + return oid.ID{}, headErr + } + + if len(objs) == 0 { + return oid.ID{}, errNoCORS + } + + sort.Slice(objs, func(i, j int) bool { + versionID1, _ := objs[i].ID() + versionID2, _ := objs[j].ID() + timestamp1 := utils.GetAttributeValue(objs[i].Attributes(), object.AttributeTimestamp) + timestamp2 := utils.GetAttributeValue(objs[j].Attributes(), object.AttributeTimestamp) + + if objs[i].CreationEpoch() != objs[j].CreationEpoch() { + return objs[i].CreationEpoch() < objs[j].CreationEpoch() + } + + if len(timestamp1) > 0 && len(timestamp2) > 0 && timestamp1 != timestamp2 { + unixTime1, err := strconv.ParseInt(timestamp1, 10, 64) + if err != nil { + return versionID1.EncodeToString() < versionID2.EncodeToString() + } + + unixTime2, err := strconv.ParseInt(timestamp2, 10, 64) + if err != nil { + return versionID1.EncodeToString() < versionID2.EncodeToString() + } + + return unixTime1 < unixTime2 + } + + return versionID1.EncodeToString() < versionID2.EncodeToString() + }) + + objID, _ := objs[len(objs)-1].ID() + return objID, nil +} + +func setCORSHeadersFromRule(c *fasthttp.RequestCtx, cors *data.CORSRule) { + c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(cors.MaxAgeSeconds)) + + if len(cors.AllowedOrigins) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, cors.AllowedOrigins[0]) + } + + if len(cors.AllowedMethods) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(cors.AllowedMethods, ", ")) + } + + if len(cors.AllowedHeaders) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, strings.Join(cors.AllowedHeaders, ", ")) + } + + if len(cors.ExposeHeaders) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(cors.ExposeHeaders, ", ")) + } + + if cors.AllowedCredentials { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + } +} + +func checkSubslice(slice []string, subSlice []string) bool { + if sliceContains(slice, wildcard) { + return true + } + if len(subSlice) > len(slice) { + return false + } + for _, r := range subSlice { + if !sliceContains(slice, r) { + return false + } + } + return true +} + +func sliceContains(slice []string, str string) bool { + for _, s := range slice { + if s == str { + return true + } + } + return false +} diff --git a/internal/handler/cors_test.go b/internal/handler/cors_test.go new file mode 100644 index 0000000..7cd7b0d --- /dev/null +++ b/internal/handler/cors_test.go @@ -0,0 +1,440 @@ +package handler + +import ( + "encoding/base64" + "encoding/xml" + "fmt" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" + "github.com/valyala/fasthttp" +) + +func TestPreflight(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName := "bucket-preflight" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + var epoch uint64 + + t.Run("CORS object", func(t *testing.T) { + for _, tc := range []struct { + name string + corsConfig *data.CORSConfiguration + requestHeaders map[string]string + expectedHeaders map[string]string + status int + }{ + { + name: "no CORS configuration", + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + fasthttp.HeaderAccessControlExposeHeaders: "", + fasthttp.HeaderAccessControlMaxAge: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + }, + status: fasthttp.StatusNotFound, + }, + { + name: "specific allowed origin", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"http://example.com"}, + AllowedMethods: []string{"GET", "HEAD"}, + AllowedHeaders: []string{"Content-Type"}, + ExposeHeaders: []string{"x-amz-*", "X-Amz-*"}, + MaxAgeSeconds: 900, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "Content-Type", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD", + fasthttp.HeaderAccessControlAllowHeaders: "Content-Type", + fasthttp.HeaderAccessControlExposeHeaders: "x-amz-*, X-Amz-*", + fasthttp.HeaderAccessControlMaxAge: "900", + fasthttp.HeaderAccessControlAllowCredentials: "true", + }, + status: fasthttp.StatusOK, + }, + { + name: "wildcard allowed origin", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + AllowedHeaders: []string{"Content-Type"}, + ExposeHeaders: []string{"x-amz-*", "X-Amz-*"}, + MaxAgeSeconds: 900, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD", + fasthttp.HeaderAccessControlAllowHeaders: "", + fasthttp.HeaderAccessControlExposeHeaders: "x-amz-*, X-Amz-*", + fasthttp.HeaderAccessControlMaxAge: "900", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + status: fasthttp.StatusOK, + }, + { + name: "not allowed header", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + AllowedHeaders: []string{"Content-Type"}, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + fasthttp.HeaderAccessControlRequestHeaders: "Authorization", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + fasthttp.HeaderAccessControlExposeHeaders: "", + fasthttp.HeaderAccessControlMaxAge: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + status: fasthttp.StatusForbidden, + }, + { + name: "empty Origin header", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + }, + }, + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + fasthttp.HeaderAccessControlExposeHeaders: "", + fasthttp.HeaderAccessControlMaxAge: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + status: fasthttp.StatusBadRequest, + }, + { + name: "empty Access-Control-Request-Method header", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + fasthttp.HeaderAccessControlExposeHeaders: "", + fasthttp.HeaderAccessControlMaxAge: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + status: fasthttp.StatusBadRequest, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.corsConfig != nil { + epoch++ + setCORSObject(t, hc, cnrID, tc.corsConfig, epoch) + } + + r := prepareCORSRequest(t, bktName, tc.requestHeaders) + hc.Handler().Preflight(r) + + require.Equal(t, tc.status, r.Response.StatusCode()) + for k, v := range tc.expectedHeaders { + require.Equal(t, v, string(r.Response.Header.Peek(k))) + } + }) + } + }) + + t.Run("CORS config", func(t *testing.T) { + hc.cfg.cors = &data.CORSRule{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + AllowedHeaders: []string{"Content-Type", "Content-Encoding"}, + ExposeHeaders: []string{"x-amz-*", "X-Amz-*"}, + MaxAgeSeconds: 900, + AllowedCredentials: true, + } + + r := prepareCORSRequest(t, bktName, map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }) + hc.Handler().Preflight(r) + + require.Equal(t, fasthttp.StatusOK, r.Response.StatusCode()) + require.Equal(t, "900", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlMaxAge))) + require.Equal(t, "*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowOrigin))) + require.Equal(t, "GET, HEAD", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowMethods))) + require.Equal(t, "Content-Type, Content-Encoding", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowHeaders))) + require.Equal(t, "x-amz-*, X-Amz-*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlExposeHeaders))) + require.Equal(t, "true", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowCredentials))) + }) +} + +func TestSetCORSHeaders(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName := "bucket-set-cors-headers" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + var epoch uint64 + + t.Run("CORS object", func(t *testing.T) { + for _, tc := range []struct { + name string + corsConfig *data.CORSConfiguration + requestHeaders map[string]string + expectedHeaders map[string]string + }{ + { + name: "empty Origin header", + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderVary: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + }, + { + name: "no CORS configuration", + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderVary: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + }, + }, + { + name: "specific allowed origin", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"http://example.com"}, + AllowedMethods: []string{"GET", "HEAD"}, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD", + fasthttp.HeaderVary: fasthttp.HeaderOrigin, + fasthttp.HeaderAccessControlAllowCredentials: "true", + }, + }, + { + name: "wildcard allowed origin, with credentials", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + }, + }, + }, + requestHeaders: func() map[string]string { + tkn := new(bearer.Token) + err = tkn.Sign(hc.key.PrivateKey) + require.NoError(t, err) + + t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) + require.NotEmpty(t, t64) + + return map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAuthorization: "Bearer " + t64, + } + }(), + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD", + fasthttp.HeaderVary: fasthttp.HeaderOrigin, + fasthttp.HeaderAccessControlAllowCredentials: "true", + }, + }, + { + name: "wildcard allowed origin, without credentials", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "*", + fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD", + fasthttp.HeaderVary: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + epoch++ + setCORSObject(t, hc, cnrID, tc.corsConfig, epoch) + r := prepareCORSRequest(t, bktName, tc.requestHeaders) + hc.Handler().SetCORSHeaders(r) + + require.Equal(t, fasthttp.StatusOK, r.Response.StatusCode()) + for k, v := range tc.expectedHeaders { + require.Equal(t, v, string(r.Response.Header.Peek(k))) + } + }) + } + }) + + t.Run("CORS config", func(t *testing.T) { + hc.cfg.cors = &data.CORSRule{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + AllowedHeaders: []string{"Content-Type", "Content-Encoding"}, + ExposeHeaders: []string{"x-amz-*", "X-Amz-*"}, + MaxAgeSeconds: 900, + AllowedCredentials: true, + } + + r := prepareCORSRequest(t, bktName, map[string]string{fasthttp.HeaderOrigin: "http://example.com"}) + hc.Handler().SetCORSHeaders(r) + + require.Equal(t, "900", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlMaxAge))) + require.Equal(t, "*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowOrigin))) + require.Equal(t, "GET, HEAD", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowMethods))) + require.Equal(t, "Content-Type, Content-Encoding", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowHeaders))) + require.Equal(t, "x-amz-*, X-Amz-*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlExposeHeaders))) + require.Equal(t, "true", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowCredentials))) + }) +} + +func TestCheckSubslice(t *testing.T) { + for _, tc := range []struct { + name string + allowed []string + actual []string + expected bool + }{ + { + name: "empty allowed slice", + allowed: []string{}, + actual: []string{"str1", "str2", "str3"}, + expected: false, + }, + { + name: "empty actual slice", + allowed: []string{"str1", "str2", "str3"}, + actual: []string{}, + expected: true, + }, + { + name: "allowed wildcard", + allowed: []string{"str", "*"}, + actual: []string{"str1", "str2", "str3"}, + expected: true, + }, + { + name: "similar allowed and actual", + allowed: []string{"str1", "str2", "str3"}, + actual: []string{"str1", "str2", "str3"}, + expected: true, + }, + { + name: "allowed actual", + allowed: []string{"str", "str1", "str2", "str4"}, + actual: []string{"str1", "str2"}, + expected: true, + }, + { + name: "not allowed actual", + allowed: []string{"str", "str1", "str2", "str4"}, + actual: []string{"str1", "str5"}, + expected: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, checkSubslice(tc.allowed, tc.actual)) + }) + } +} + +func setCORSObject(t *testing.T, hc *handlerContext, cnrID cid.ID, corsConfig *data.CORSConfiguration, epoch uint64) { + payload, err := xml.Marshal(corsConfig) + require.NoError(t, err) + + a := object.NewAttribute() + a.SetKey(object.AttributeFilePath) + a.SetValue(fmt.Sprintf(corsFilePathTemplate, cnrID)) + + objID := oidtest.ID() + obj := object.New() + obj.SetAttributes(*a) + obj.SetOwnerID(hc.owner) + obj.SetPayload(payload) + obj.SetPayloadSize(uint64(len(payload))) + obj.SetContainerID(hc.corsCnr) + obj.SetID(objID) + obj.SetCreationEpoch(epoch) + + var addr oid.Address + addr.SetObject(objID) + addr.SetContainer(hc.corsCnr) + + hc.frostfs.SetObject(addr, obj) +} diff --git a/internal/handler/frostfs_mock.go b/internal/handler/frostfs_mock.go index b60915e..7d72ad9 100644 --- a/internal/handler/frostfs_mock.go +++ b/internal/handler/frostfs_mock.go @@ -52,6 +52,10 @@ func (t *TestFrostFS) SetContainer(cnrID cid.ID, cnr *container.Container) { t.containers[cnrID.EncodeToString()] = cnr } +func (t *TestFrostFS) SetObject(addr oid.Address, obj *object.Object) { + t.objects[addr.EncodeToString()] = obj +} + // AllowUserOperation grants access to object operations. // Empty userID and objID means any user and object respectively. func (t *TestFrostFS) AllowUserOperation(cnrID cid.ID, userID user.ID, op acl.Op, objID oid.ID) { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 179cf60..48f8f55 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -36,6 +36,8 @@ type Config interface { BufferMaxSizeForPut() uint64 NamespaceHeader() string EnableFilepathFallback() bool + FormContainerZone(string) string + CORS() *data.CORSRule } // PrmContainer groups parameters of FrostFS.Container operation. @@ -158,7 +160,7 @@ type FrostFS interface { } type ContainerResolver interface { - Resolve(ctx context.Context, name string) (*cid.ID, error) + Resolve(ctx context.Context, zone, name string) (*cid.ID, error) } type Handler struct { @@ -170,14 +172,18 @@ type Handler struct { tree layer.TreeService cache *cache.BucketCache workerPool *ants.Pool + corsCnrID cid.ID + corsCache *cache.CORSCache } type AppParams struct { - Logger *zap.Logger - FrostFS FrostFS - Owner *user.ID - Resolver ContainerResolver - Cache *cache.BucketCache + Logger *zap.Logger + FrostFS FrostFS + Owner *user.ID + Resolver ContainerResolver + Cache *cache.BucketCache + CORSCnrID cid.ID + CORSCache *cache.CORSCache } func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler { @@ -190,6 +196,8 @@ func New(params *AppParams, config Config, tree layer.TreeService, workerPool *a tree: tree, cache: params.Cache, workerPool: workerPool, + corsCnrID: params.CORSCnrID, + corsCache: params.CORSCache, } } @@ -352,7 +360,14 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci cnrID := new(cid.ID) err := cnrID.DecodeString(containerID) if err != nil { - cnrID, err = h.containerResolver.Resolve(ctx, containerID) + var namespace string + namespace, err = middleware.GetNamespace(ctx) + if err != nil { + return nil, err + } + + zone := h.config.FormContainerZone(namespace) + cnrID, err = h.containerResolver.Resolve(ctx, zone, containerID) if err != nil && strings.Contains(err.Error(), "not found") { err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error()) } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index ab2cd9f..3a81c50 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -16,7 +16,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -61,6 +63,7 @@ func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*d type configMock struct { additionalSearch bool + cors *data.CORSRule } func (c *configMock) DefaultTimestamp() bool { @@ -99,9 +102,18 @@ func (c *configMock) EnableFilepathFallback() bool { return c.additionalSearch } +func (c *configMock) FormContainerZone(string) string { + return v2container.SysAttributeZoneDefault +} + +func (c *configMock) CORS() *data.CORSRule { + return c.cors +} + type handlerContext struct { - key *keys.PrivateKey - owner user.ID + key *keys.PrivateKey + owner user.ID + corsCnr cid.ID h *Handler frostfs *TestFrostFS @@ -131,10 +143,12 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { testFrostFS := NewTestFrostFS(key) testResolver := &resolver.Resolver{Name: "test_resolver"} - testResolver.SetResolveFunc(func(_ context.Context, name string) (*cid.ID, error) { + testResolver.SetResolveFunc(func(_ context.Context, _, name string) (*cid.ID, error) { return testFrostFS.ContainerID(name) }) + cnrID := createCORSContainer(owner, testFrostFS) + params := &AppParams{ Logger: logger, FrostFS: testFrostFS, @@ -145,6 +159,12 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { Lifetime: 1, Logger: logger, }, false), + CORSCnrID: cnrID, + CORSCache: cache.NewCORSCache(&cache.Config{ + Size: 1, + Lifetime: 1, + Logger: logger, + }), } treeMock := newTreeService() @@ -159,6 +179,7 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { return &handlerContext{ key: key, owner: owner, + corsCnr: cnrID, h: handler, frostfs: testFrostFS, tree: treeMock, @@ -166,6 +187,20 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { }, nil } +func createCORSContainer(owner user.ID, frostfs *TestFrostFS) cid.ID { + var cnr container.Container + cnr.Init() + cnr.SetOwner(owner) + + cnrID := cidtest.ID() + frostfs.SetContainer(cnrID, &cnr) + frostfs.AllowUserOperation(cnrID, owner, acl.OpObjectSearch, oid.ID{}) + frostfs.AllowUserOperation(cnrID, owner, acl.OpObjectHead, oid.ID{}) + frostfs.AllowUserOperation(cnrID, owner, acl.OpObjectGet, oid.ID{}) + + return cnrID +} + func (hc *handlerContext) prepareContainer(name string, basicACL acl.Basic) (cid.ID, *container.Container, error) { var pp netmap.PlacementPolicy err := pp.DecodeString("REP 1") @@ -486,6 +521,25 @@ func prepareGetRequest(ctx context.Context, bucket, objID string) *fasthttp.Requ return r } +func prepareCORSRequest(t *testing.T, bucket string, headers map[string]string) *fasthttp.RequestCtx { + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + r := new(fasthttp.RequestCtx) + r.SetUserValue("cid", bucket) + + for k, v := range headers { + r.Request.Header.Set(k, v) + } + + ctx, err := tokens.StoreBearerTokenAppCtx(ctx, r) + require.NoError(t, err) + + utils.SetContextToRequest(ctx, r) + + return r +} + func prepareGetByAttributeRequest(ctx context.Context, bucket, attrKey, attrVal string) *fasthttp.RequestCtx { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index f8f1da9..3166f98 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -72,41 +72,47 @@ const ( TagsLogConfigWontBeUpdated = "tags log config won't be updated" FailedToReadIndexPageTemplate = "failed to read index page template" SetCustomIndexPageTemplate = "set custom index page template" + CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info" ) // Log messages with the "datapath" tag. const ( - CouldntParseCreationDate = "couldn't parse creation date" - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" - FailedToAddObjectToArchive = "failed to add object to archive" - CloseZipWriter = "close zip writer" - IgnorePartEmptyFormName = "ignore part, empty form name" - IgnorePartEmptyFilename = "ignore part, empty filename" - CouldNotParseClientTime = "could not parse client time" - CouldNotPrepareExpirationHeader = "could not prepare expiration header" - CouldNotEncodeResponse = "could not encode response" - AddAttributeToResultObject = "add attribute to result object" - Request = "request" - CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" - CouldntPutBucketIntoCache = "couldn't put bucket info into cache" - FailedToIterateOverResponse = "failed to iterate over search response" - InvalidCacheEntryType = "invalid cache entry type" - FailedToUnescapeQuery = "failed to unescape query" - CouldntCacheNetmap = "couldn't cache netmap" - FailedToCloseReader = "failed to close reader" - FailedToFilterHeaders = "failed to filter headers" - FailedToReadFileFromTar = "failed to read file from tar" - FailedToGetAttributes = "failed to get attributes" - CloseGzipWriter = "close gzip writer" - CloseTarWriter = "close tar writer" - FailedToCreateGzipReader = "failed to create gzip reader" - GzipReaderSelected = "gzip reader selected" - CouldNotReceiveMultipartForm = "could not receive multipart/form" - ObjectsNotFound = "objects not found" - IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" - CouldNotGetBucket = "could not get bucket" - CouldNotResolveContainerID = "could not resolve container id" - FailedToSumbitTaskToPool = "failed to submit task to pool" + CouldntParseCreationDate = "couldn't parse creation date" + CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" + FailedToAddObjectToArchive = "failed to add object to archive" + CloseZipWriter = "close zip writer" + IgnorePartEmptyFormName = "ignore part, empty form name" + IgnorePartEmptyFilename = "ignore part, empty filename" + CouldNotParseClientTime = "could not parse client time" + CouldNotPrepareExpirationHeader = "could not prepare expiration header" + CouldNotEncodeResponse = "could not encode response" + AddAttributeToResultObject = "add attribute to result object" + Request = "request" + CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" + CouldntPutBucketIntoCache = "couldn't put bucket info into cache" + FailedToIterateOverResponse = "failed to iterate over search response" + InvalidCacheEntryType = "invalid cache entry type" + FailedToUnescapeQuery = "failed to unescape query" + CouldntCacheNetmap = "couldn't cache netmap" + FailedToCloseReader = "failed to close reader" + FailedToFilterHeaders = "failed to filter headers" + FailedToReadFileFromTar = "failed to read file from tar" + FailedToGetAttributes = "failed to get attributes" + CloseGzipWriter = "close gzip writer" + CloseTarWriter = "close tar writer" + FailedToCreateGzipReader = "failed to create gzip reader" + GzipReaderSelected = "gzip reader selected" + CouldNotReceiveMultipartForm = "could not receive multipart/form" + ObjectsNotFound = "objects not found" + IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" + CouldNotGetBucket = "could not get bucket" + CouldNotResolveContainerID = "could not resolve container id" + FailedToSumbitTaskToPool = "failed to submit task to pool" + CouldNotGetCORSConfiguration = "could not get cors configuration" + EmptyOriginRequestHeader = "empty Origin request header" + EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header" + CORSRuleWasNotMatched = "cors rule was not matched" + CouldntCacheCors = "couldn't cache cors" ) // Log messages with the "external_storage" tag. diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index 9115930..4cf45a4 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -10,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -21,8 +20,6 @@ import ( "google.golang.org/grpc/status" ) -const clientIOTag = "client" - // FrostFS represents virtual connection to the FrostFS network. // It is used to provide an interface to dependent packages // which work with FrostFS. @@ -70,7 +67,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) prmPut.UseBearer(*prm.BearerToken) } - idObj, err := x.pool.PutObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmPut) + idObj, err := x.pool.PutObject(ctx, prmPut) if err != nil { return oid.ID{}, handleObjectError("save object via connection pool", err) } @@ -103,7 +100,7 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o prmHead.UseBearer(*prm.BearerToken) } - res, err := x.pool.HeadObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmHead) + res, err := x.pool.HeadObject(ctx, prmHead) if err != nil { return nil, handleObjectError("read object header via connection pool", err) } @@ -123,7 +120,7 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han prmGet.UseBearer(*prm.BearerToken) } - res, err := x.pool.GetObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmGet) + res, err := x.pool.GetObject(ctx, prmGet) if err != nil { return nil, handleObjectError("init full object reading via connection pool", err) } @@ -148,7 +145,7 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) ( prmRange.UseBearer(*prm.BearerToken) } - res, err := x.pool.ObjectRange(qostagging.ContextWithIOTag(ctx, clientIOTag), prmRange) + res, err := x.pool.ObjectRange(ctx, prmRange) if err != nil { return nil, handleObjectError("init payload range reading via connection pool", err) } @@ -169,7 +166,7 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch prmSearch.UseBearer(*prm.BearerToken) } - res, err := x.pool.SearchObjects(qostagging.ContextWithIOTag(ctx, clientIOTag), prmSearch) + res, err := x.pool.SearchObjects(ctx, prmSearch) if err != nil { return nil, handleObjectError("init object search via connection pool", err) } diff --git a/internal/service/frostfs/tree_pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go index 89afc3c..410acda 100644 --- a/internal/service/frostfs/tree_pool_wrapper.go +++ b/internal/service/frostfs/tree_pool_wrapper.go @@ -10,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apitree "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/tree" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" ) @@ -62,7 +61,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([ BearerToken: getBearer(ctx), } - nodes, err := w.p.GetNodes(qostagging.ContextWithIOTag(ctx, clientIOTag), poolPrm) + nodes, err := w.p.GetNodes(ctx, poolPrm) if err != nil { return nil, handleError(err) } @@ -121,7 +120,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, poolPrm.RootID = nil } - subTreeReader, err := w.p.GetSubTree(qostagging.ContextWithIOTag(ctx, clientIOTag), poolPrm) + subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) if err != nil { return nil, handleError(err) } diff --git a/resolver/resolver.go b/resolver/resolver.go index e7615d4..6d7c5d5 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -6,7 +6,7 @@ import ( "fmt" "sync" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" @@ -29,14 +29,9 @@ type FrostFS interface { SystemDNS(context.Context) (string, error) } -type Settings interface { - FormContainerZone(ns string) (zone string, isDefault bool) -} - type Config struct { FrostFS FrostFS RPCAddress string - Settings Settings } type ContainerResolver struct { @@ -46,15 +41,15 @@ type ContainerResolver struct { type Resolver struct { Name string - resolve func(context.Context, string) (*cid.ID, error) + resolve func(context.Context, string, string) (*cid.ID, error) } -func (r *Resolver) SetResolveFunc(fn func(context.Context, string) (*cid.ID, error)) { +func (r *Resolver) SetResolveFunc(fn func(context.Context, string, string) (*cid.ID, error)) { r.resolve = fn } -func (r *Resolver) Resolve(ctx context.Context, name string) (*cid.ID, error) { - return r.resolve(ctx, name) +func (r *Resolver) Resolve(ctx context.Context, zone, name string) (*cid.ID, error) { + return r.resolve(ctx, zone, name) } func NewContainerResolver(resolverNames []string, cfg *Config) (*ContainerResolver, error) { @@ -81,13 +76,13 @@ func createResolvers(resolverNames []string, cfg *Config) ([]*Resolver, error) { return resolvers, nil } -func (r *ContainerResolver) Resolve(ctx context.Context, cnrName string) (*cid.ID, error) { +func (r *ContainerResolver) Resolve(ctx context.Context, cnrZone, cnrName string) (*cid.ID, error) { r.mu.RLock() defer r.mu.RUnlock() var err error for _, resolver := range r.resolvers { - cnrID, resolverErr := resolver.Resolve(ctx, cnrName) + cnrID, resolverErr := resolver.Resolve(ctx, cnrZone, cnrName) if resolverErr != nil { resolverErr = fmt.Errorf("%s: %w", resolver.Name, resolverErr) if err == nil { @@ -141,34 +136,25 @@ func (r *ContainerResolver) equals(resolverNames []string) bool { func newResolver(name string, cfg *Config) (*Resolver, error) { switch name { case DNSResolver: - return NewDNSResolver(cfg.FrostFS, cfg.Settings) + return NewDNSResolver(cfg.FrostFS) case NNSResolver: - return NewNNSResolver(cfg.RPCAddress, cfg.Settings) + return NewNNSResolver(cfg.RPCAddress) default: return nil, fmt.Errorf("unknown resolver: %s", name) } } -func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) { +func NewDNSResolver(frostFS FrostFS) (*Resolver, error) { if frostFS == nil { return nil, fmt.Errorf("pool must not be nil for DNS resolver") } - if settings == nil { - return nil, fmt.Errorf("resolver settings must not be nil for DNS resolver") - } var dns ns.DNS - resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) { + resolveFunc := func(ctx context.Context, zone, name string) (*cid.ID, error) { var err error - namespace, err := middleware.GetNamespace(ctx) - if err != nil { - return nil, err - } - - zone, isDefault := settings.FormContainerZone(namespace) - if isDefault { + if zone == v2container.SysAttributeZoneDefault { zone, err = frostFS.SystemDNS(ctx) if err != nil { return nil, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err) @@ -190,13 +176,10 @@ func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) { }, nil } -func NewNNSResolver(rpcAddress string, settings Settings) (*Resolver, error) { +func NewNNSResolver(rpcAddress string) (*Resolver, error) { if rpcAddress == "" { return nil, fmt.Errorf("rpc address must not be empty for NNS resolver") } - if settings == nil { - return nil, fmt.Errorf("resolver settings must not be nil for NNS resolver") - } var nns ns.NNS @@ -204,16 +187,9 @@ func NewNNSResolver(rpcAddress string, settings Settings) (*Resolver, error) { return nil, fmt.Errorf("could not dial nns: %w", err) } - resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) { + resolveFunc := func(_ context.Context, zone, name string) (*cid.ID, error) { var d container.Domain d.SetName(name) - - namespace, err := middleware.GetNamespace(ctx) - if err != nil { - return nil, err - } - - zone, _ := settings.FormContainerZone(namespace) d.SetZone(zone) cnrID, err := nns.ResolveContainerDomain(d) diff --git a/utils/attributes.go b/utils/attributes.go index 4d277a9..55fadaa 100644 --- a/utils/attributes.go +++ b/utils/attributes.go @@ -11,6 +11,8 @@ import ( "time" "unicode" "unicode/utf8" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) type EpochDurations struct { @@ -256,3 +258,12 @@ func (t systemTransformer) updateExpirationHeader(headers map[string]string, dur headers[t.expirationEpochAttr()] = strconv.FormatUint(expirationEpoch, 10) } + +func GetAttributeValue(attrs []object.Attribute, key string) string { + for _, attr := range attrs { + if attr.Key() == key { + return attr.Value() + } + } + return "" +} From d670983df4453b91fa96cc0ab5400d6e7e89fcd2 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Mon, 10 Feb 2025 18:30:16 +0300 Subject: [PATCH 72/86] [#208] govulncheck: Fix minor toolchain updates for good Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/vulncheck.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 5cb6e73..5fb9dc5 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -16,7 +16,8 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22.12' + go-version: '1.22' + check-latest: true - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest From 0f73da258bb0974097a2cf86242cc1c10ddd4198 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 20 Mar 2025 18:39:30 +0300 Subject: [PATCH 73/86] [#223] Bump frostfs-sdk-go Contains: * more detailed pool errors * disabled service config query in gRPC client Signed-off-by: Alex Vanin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0ace5f2..31cf242 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 diff --git a/go.sum b/go.sum index a2121ab..6050ad6 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,8 @@ git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe h1:81gDNdWNLP24oMQukRiCE9R1wGSh0l0dRq3F1W+Oesc= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc h1:fS6Yp4GvI+C22UrWz9oqJXwvQw5Q6SmADIY4H9eIQsc= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 458bf933fcef2f782d91d60c70fd5b89a64ff859 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 3 Mar 2025 18:06:41 +0300 Subject: [PATCH 74/86] [#191] Refactor error handling and logging Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 4 +- docs/api.md | 26 ++-- internal/handler/browse.go | 22 ++- internal/handler/cors.go | 70 ++++----- internal/handler/download.go | 133 ++++++++---------- internal/handler/handler.go | 107 ++++++-------- internal/handler/handler_test.go | 2 +- internal/handler/head.go | 40 +++--- internal/handler/multipart.go | 4 +- internal/handler/reader.go | 27 ++-- internal/handler/upload.go | 109 ++++++-------- internal/handler/utils.go | 83 ++++------- internal/logs/logs.go | 30 ++-- internal/service/frostfs/frostfs.go | 29 ++-- internal/service/frostfs/frostfs_test.go | 14 +- internal/service/frostfs/tree_pool_wrapper.go | 8 +- tree/tree.go | 17 ++- 17 files changed, 327 insertions(+), 398 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index c75f9d8..de186fb 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -724,12 +724,12 @@ func (a *app) stopServices() { } func (a *app) configureRouter(workerPool *ants.Pool) { - a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) + a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool) r := router.New() r.RedirectTrailingSlash = true r.NotFound = func(r *fasthttp.RequestCtx) { - handler.ResponseError(r, "Not found", fasthttp.StatusNotFound) + handler.ResponseError(r, "Route Not found", fasthttp.StatusNotFound) } r.MethodNotAllowed = func(r *fasthttp.RequestCtx) { handler.ResponseError(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) diff --git a/docs/api.md b/docs/api.md index d099915..698e9b1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -94,6 +94,8 @@ The `filename` field from the multipart form will be set as `FileName` attribute |--------|----------------------------------------------| | 200 | Object created successfully. | | 400 | Some error occurred during object uploading. | +| 403 | Access denied. | +| 409 | Can not upload object due to quota reached. | ## Get object @@ -141,6 +143,7 @@ Get an object (payload and attributes) by an address. |--------|------------------------------------------------| | 200 | Object got successfully. | | 400 | Some error occurred during object downloading. | +| 403 | Access denied. | | 404 | Container or object not found. | ###### Body @@ -183,6 +186,7 @@ Get an object attributes by an address. |--------|---------------------------------------------------| | 200 | Object head successfully. | | 400 | Some error occurred during object HEAD operation. | +| 403 | Access denied. | | 404 | Container or object not found. | ## Search object @@ -233,6 +237,7 @@ If more than one object is found, an arbitrary one will be returned. |--------|------------------------------------------------| | 200 | Object got successfully. | | 400 | Some error occurred during object downloading. | +| 403 | Access denied. | | 404 | Container or object not found. | #### HEAD @@ -269,6 +274,7 @@ If more than one object is found, an arbitrary one will be used to get attribute |--------|---------------------------------------| | 200 | Object head successfully. | | 400 | Some error occurred during operation. | +| 403 | Access denied. | | 404 | Container or object not found. | ## Download archive @@ -304,16 +310,16 @@ Archive can be compressed (see http-gw [configuration](gate-configuration.md#arc ###### Headers -| Header | Description | -|-----------------------|-------------------------------------------------------------------------------------------------------------------| -| `Content-Disposition` | Indicate how to browsers should treat file (`attachment`). Set `filename` as `archive.zip`. | -| `Content-Type` | Indicate content type of object. Set to `application/zip` | +| Header | Description | +|-----------------------|---------------------------------------------------------------------------------------------| +| `Content-Disposition` | Indicate how to browsers should treat file (`attachment`). Set `filename` as `archive.zip`. | +| `Content-Type` | Indicate content type of object. Set to `application/zip` | ###### Status codes -| Status | Description | -|--------|-----------------------------------------------------| -| 200 | Object got successfully. | -| 400 | Some error occurred during object downloading. | -| 404 | Container or objects not found. | -| 500 | Some inner error (e.g. error on streaming objects). | +| Status | Description | +|--------|------------------------------------------------| +| 200 | Object got successfully. | +| 400 | Some error occurred during object downloading. | +| 403 | Access denied. | +| 404 | Container or objects not found. | diff --git a/internal/handler/browse.go b/internal/handler/browse.go index 2d0e34d..ebe9004 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -223,7 +223,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck return nil, err } - log := utils.GetReqLogOrDefault(ctx, h.log) + log := h.reqLogger(ctx) dirs := make(map[string]struct{}) result := &GetObjectsResponse{ objects: make([]ResponseObject, 0, 100), @@ -258,7 +258,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re go func() { defer close(res) - log := utils.GetReqLogOrDefault(ctx, h.log).With( + log := h.reqLogger(ctx).With( zap.String("cid", cnrID.EncodeToString()), zap.String("path", basePath), ) @@ -273,7 +273,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re }) if err != nil { wg.Done() - log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath)) + log.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath)) } select { case <-ctx.Done(): @@ -328,20 +328,18 @@ type browseParams struct { listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error) } -func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) { +func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p browseParams) { const S3Protocol = "s3" const FrostfsProtocol = "frostfs" - ctx := utils.GetContextFromRequest(c) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With( + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( zap.String("bucket", p.bucketInfo.Name), zap.String("container", p.bucketInfo.CID.EncodeToString()), zap.String("prefix", p.prefix), - ) + )) resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToListObjects, err) return } @@ -360,7 +358,7 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) { "parentDir": parentDir, }).Parse(h.config.IndexPageTemplate()) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToParseTemplate, err) return } bucketName := p.bucketInfo.Name @@ -369,14 +367,14 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) { bucketName = p.bucketInfo.CID.EncodeToString() protocol = FrostfsProtocol } - if err = tmpl.Execute(c, &BrowsePageData{ + if err = tmpl.Execute(req, &BrowsePageData{ Container: bucketName, Prefix: p.prefix, Objects: objects, Protocol: protocol, HasErrors: resp.hasErrors, }); err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToExecuteTemplate, err) return } } diff --git a/internal/handler/cors.go b/internal/handler/cors.go index 234ef2a..d77ae02 100644 --- a/internal/handler/cors.go +++ b/internal/handler/cors.go @@ -30,32 +30,32 @@ const ( var errNoCORS = errors.New("no CORS objects found") -func (h *Handler) Preflight(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.Preflight") +func (h *Handler) Preflight(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.Preflight") defer span.End() ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) - cidParam, _ := c.UserValue("cid").(string) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) + cidParam, _ := req.UserValue("cid").(string) + reqLog := h.reqLogger(ctx) log := reqLog.With(zap.String("cid", cidParam)) - origin := c.Request.Header.Peek(fasthttp.HeaderOrigin) + origin := req.Request.Header.Peek(fasthttp.HeaderOrigin) if len(origin) == 0 { log.Error(logs.EmptyOriginRequestHeader, logs.TagField(logs.TagDatapath)) - ResponseError(c, "Origin request header needed", fasthttp.StatusBadRequest) + ResponseError(req, "Origin request header needed", fasthttp.StatusBadRequest) return } - method := c.Request.Header.Peek(fasthttp.HeaderAccessControlRequestMethod) + method := req.Request.Header.Peek(fasthttp.HeaderAccessControlRequestMethod) if len(method) == 0 { log.Error(logs.EmptyAccessControlRequestMethodHeader, logs.TagField(logs.TagDatapath)) - ResponseError(c, "Access-Control-Request-Method request header needed", fasthttp.StatusBadRequest) + ResponseError(req, "Access-Control-Request-Method request header needed", fasthttp.StatusBadRequest) return } corsRule := h.config.CORS() if corsRule != nil { - setCORSHeadersFromRule(c, corsRule) + setCORSHeadersFromRule(req, corsRule) return } @@ -66,12 +66,12 @@ func (h *Handler) Preflight(c *fasthttp.RequestCtx) { if errors.Is(err, errNoCORS) { status = fasthttp.StatusNotFound } - ResponseError(c, "could not get CORS configuration: "+err.Error(), status) + ResponseError(req, "could not get CORS configuration: "+err.Error(), status) return } var headers []string - requestHeaders := c.Request.Header.Peek(fasthttp.HeaderAccessControlRequestHeaders) + requestHeaders := req.Request.Header.Peek(fasthttp.HeaderAccessControlRequestHeaders) if len(requestHeaders) > 0 { headers = strings.Split(string(requestHeaders), ", ") } @@ -84,19 +84,19 @@ func (h *Handler) Preflight(c *fasthttp.RequestCtx) { if !checkSubslice(rule.AllowedHeaders, headers) { continue } - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) if headers != nil { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, string(requestHeaders)) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, string(requestHeaders)) } if rule.ExposeHeaders != nil { - c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(rule.ExposeHeaders, ", ")) + req.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(rule.ExposeHeaders, ", ")) } if rule.MaxAgeSeconds > 0 || rule.MaxAgeSeconds == -1 { - c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(rule.MaxAgeSeconds)) + req.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(rule.MaxAgeSeconds)) } if o != wildcard { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") } return } @@ -105,26 +105,26 @@ func (h *Handler) Preflight(c *fasthttp.RequestCtx) { } } log.Error(logs.CORSRuleWasNotMatched, logs.TagField(logs.TagDatapath)) - ResponseError(c, "Forbidden", fasthttp.StatusForbidden) + ResponseError(req, "Forbidden", fasthttp.StatusForbidden) } -func (h *Handler) SetCORSHeaders(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.SetCORSHeaders") +func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.SetCORSHeaders") defer span.End() - origin := c.Request.Header.Peek(fasthttp.HeaderOrigin) + origin := req.Request.Header.Peek(fasthttp.HeaderOrigin) if len(origin) == 0 { return } ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) - cidParam, _ := c.UserValue("cid").(string) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) + cidParam, _ := req.UserValue("cid").(string) + reqLog := h.reqLogger(ctx) log := reqLog.With(zap.String("cid", cidParam)) corsRule := h.config.CORS() if corsRule != nil { - setCORSHeadersFromRule(c, corsRule) + setCORSHeadersFromRule(req, corsRule) return } @@ -143,26 +143,26 @@ func (h *Handler) SetCORSHeaders(c *fasthttp.RequestCtx) { for _, o := range rule.AllowedOrigins { if o == string(origin) { for _, m := range rule.AllowedMethods { - if m == string(c.Method()) { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") - c.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) + if m == string(req.Method()) { + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + req.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) return } } } if o == wildcard { for _, m := range rule.AllowedMethods { - if m == string(c.Method()) { + if m == string(req.Method()) { if withCredentials { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") - c.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + req.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) } else { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, o) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, o) } - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) return } } diff --git a/internal/handler/download.go b/internal/handler/download.go index b398a54..114bf34 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -25,43 +25,38 @@ import ( ) // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. -func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByAddressOrBucketName") +func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.DownloadByAddressOrBucketName") defer span.End() - utils.SetContextToRequest(ctx, c) - cidParam := c.UserValue("cid").(string) - oidParam := c.UserValue("oid").(string) - downloadParam := c.QueryArgs().GetBool("download") + cidParam := req.UserValue("cid").(string) + oidParam := req.UserValue("oid").(string) + downloadParam := req.QueryArgs().GetBool("download") - log := utils.GetReqLogOrDefault(ctx, h.log).With( + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( zap.String("cid", cidParam), zap.String("oid", oidParam), - ) + )) - bktInfo, err := h.getBucketInfo(ctx, cidParam, log) + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { - log.Error(logs.FailedToCheckIfSettingsNodeExist, zap.String("cid", bktInfo.CID.String()), - zap.Error(checkS3Err), logs.TagField(logs.TagExternalStorageTree)) - logAndSendBucketError(c, log, checkS3Err) + h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) return } - req := newRequest(c, log) - var objID oid.ID if checkS3Err == nil && shouldDownload(oidParam, downloadParam) { h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile) } else if err = objID.DecodeString(oidParam); err == nil { h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile) } else { - h.browseIndex(c, checkS3Err != nil) + h.browseIndex(ctx, req, cidParam, oidParam, checkS3Err != nil) } } @@ -70,12 +65,11 @@ func shouldDownload(oidParam string, downloadParam bool) bool { } // DownloadByAttribute handles attribute-based download requests. -func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByAttribute") +func (h *Handler) DownloadByAttribute(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.DownloadByAttribute") defer span.End() - utils.SetContextToRequest(ctx, c) - h.byAttribute(c, h.receiveFile) + h.byAttribute(ctx, req, h.receiveFile) } func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) { @@ -95,31 +89,33 @@ func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op } // DownloadZip handles zip by prefix requests. -func (h *Handler) DownloadZip(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadZip") +func (h *Handler) DownloadZip(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.DownloadZip") defer span.End() - utils.SetContextToRequest(ctx, c) - scid, _ := c.UserValue("cid").(string) + scid, _ := req.UserValue("cid").(string) + prefix, _ := req.UserValue("prefix").(string) - log := utils.GetReqLogOrDefault(ctx, h.log) - bktInfo, err := h.getBucketInfo(ctx, scid, log) + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", scid), zap.String("prefix", prefix))) + + bktInfo, err := h.getBucketInfo(ctx, scid) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } - resSearch, err := h.searchObjectsByPrefix(c, log, bktInfo.CID) + + resSearch, err := h.searchObjectsByPrefix(ctx, bktInfo.CID, prefix) if err != nil { return } - c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") - c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") + req.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") + req.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") - c.SetBodyStreamWriter(h.getZipResponseWriter(ctx, log, resSearch, bktInfo)) + req.SetBodyStreamWriter(h.getZipResponseWriter(ctx, resSearch, bktInfo)) } -func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { +func (h *Handler) getZipResponseWriter(ctx context.Context, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { return func(w *bufio.Writer) { defer resSearch.Close() @@ -127,20 +123,20 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res zipWriter := zip.NewWriter(w) var objectsWritten int - errIter := resSearch.Iterate(h.putObjectToArchive(ctx, log, bktInfo.CID, buf, + errIter := resSearch.Iterate(h.putObjectToArchive(ctx, bktInfo.CID, buf, func(obj *object.Object) (io.Writer, error) { objectsWritten++ return h.createZipFile(zipWriter, obj) }), ) if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) return } else if objectsWritten == 0 { - log.Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) } if err := zipWriter.Close(); err != nil { - log.Error(logs.CloseZipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.CloseZipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } } } @@ -164,31 +160,33 @@ func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, } // DownloadTar forms tar.gz from objects by prefix. -func (h *Handler) DownloadTar(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadTar") +func (h *Handler) DownloadTar(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.DownloadTar") defer span.End() - utils.SetContextToRequest(ctx, c) - scid, _ := c.UserValue("cid").(string) + scid, _ := req.UserValue("cid").(string) + prefix, _ := req.UserValue("prefix").(string) - log := utils.GetReqLogOrDefault(ctx, h.log) - bktInfo, err := h.getBucketInfo(ctx, scid, log) + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", scid), zap.String("prefix", prefix))) + + bktInfo, err := h.getBucketInfo(ctx, scid) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } - resSearch, err := h.searchObjectsByPrefix(c, log, bktInfo.CID) + + resSearch, err := h.searchObjectsByPrefix(ctx, bktInfo.CID, prefix) if err != nil { return } - c.Response.Header.Set(fasthttp.HeaderContentType, "application/gzip") - c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.tar.gz\"") + req.Response.Header.Set(fasthttp.HeaderContentType, "application/gzip") + req.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.tar.gz\"") - c.SetBodyStreamWriter(h.getTarResponseWriter(ctx, log, resSearch, bktInfo)) + req.SetBodyStreamWriter(h.getTarResponseWriter(ctx, resSearch, bktInfo)) } -func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { +func (h *Handler) getTarResponseWriter(ctx context.Context, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { return func(w *bufio.Writer) { defer resSearch.Close() @@ -203,26 +201,26 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res defer func() { if err := tarWriter.Close(); err != nil { - log.Error(logs.CloseTarWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.CloseTarWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } if err := gzipWriter.Close(); err != nil { - log.Error(logs.CloseGzipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.CloseGzipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } }() var objectsWritten int buf := make([]byte, 3<<20) // the same as for upload - errIter := resSearch.Iterate(h.putObjectToArchive(ctx, log, bktInfo.CID, buf, + errIter := resSearch.Iterate(h.putObjectToArchive(ctx, bktInfo.CID, buf, func(obj *object.Object) (io.Writer, error) { objectsWritten++ return h.createTarFile(tarWriter, obj) }), ) if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) } else if objectsWritten == 0 { - log.Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) } } } @@ -240,9 +238,9 @@ func (h *Handler) createTarFile(tw *tar.Writer, obj *object.Object) (io.Writer, }) } -func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID cid.ID, buf []byte, createArchiveHeader func(obj *object.Object) (io.Writer, error)) func(id oid.ID) bool { +func (h *Handler) putObjectToArchive(ctx context.Context, cnrID cid.ID, buf []byte, createArchiveHeader func(obj *object.Object) (io.Writer, error)) func(id oid.ID) bool { return func(id oid.ID) bool { - log = log.With(zap.String("oid", id.EncodeToString())) + logger := h.reqLogger(ctx).With(zap.String("oid", id.EncodeToString())) prm := PrmObjectGet{ PrmAuth: PrmAuth{ @@ -253,18 +251,18 @@ func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID resGet, err := h.frostfs.GetObject(ctx, prm) if err != nil { - log.Error(logs.FailedToGetObject, zap.Error(err), logs.TagField(logs.TagExternalStorage)) + logger.Error(logs.FailedToGetObject, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return false } fileWriter, err := createArchiveHeader(&resGet.Header) if err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) + logger.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) return false } if err = writeToArchive(resGet, fileWriter, buf); err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) + logger.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) return false } @@ -272,28 +270,17 @@ func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID } } -func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, cnrID cid.ID) (ResObjectSearch, error) { - scid, _ := c.UserValue("cid").(string) - prefix, _ := c.UserValue("prefix").(string) - - ctx := utils.GetContextFromRequest(c) - +func (h *Handler) searchObjectsByPrefix(ctx context.Context, cnrID cid.ID, prefix string) (ResObjectSearch, error) { prefix, err := url.QueryUnescape(prefix) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), - zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) - return nil, err + return nil, fmt.Errorf("unescape prefix: %w", err) } - log = log.With(zap.String("cid", scid), zap.String("prefix", prefix)) - resSearch, err := h.search(ctx, cnrID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage)) - ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) - return nil, err + return nil, fmt.Errorf("search objects by prefix: %w", err) } + return resSearch, nil } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 48f8f55..a982bc2 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -16,7 +16,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -144,6 +143,10 @@ var ( ErrGatewayTimeout = errors.New("gateway timeout") // ErrQuotaLimitReached is returned from FrostFS in case of quota exceeded. ErrQuotaLimitReached = errors.New("quota limit reached") + // ErrContainerNotFound is returned from FrostFS in case of container was not found. + ErrContainerNotFound = errors.New("container not found") + // ErrObjectNotFound is returned from FrostFS in case of object was not found. + ErrObjectNotFound = errors.New("object not found") ) // FrostFS represents virtual connection to FrostFS network. @@ -203,7 +206,7 @@ func New(params *AppParams, config Config, tree layer.TreeService, workerPool *a // byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) { +func (h *Handler) byNativeAddress(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, objID oid.ID, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { ctx, span := tracing.StartSpanFromContext(ctx, "handler.byNativeAddress") defer span.End() @@ -213,72 +216,59 @@ func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID // byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that // resolves object address from S3-like path /. -func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) { +func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path") defer span.End() - c, log := req.RequestCtx, req.log - foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) if err != nil { - log.Error(logs.FailedToGetLatestVersionOfObject, zap.Error(err), zap.String("cid", cnrID.String()), - zap.String("path", path), logs.TagField(logs.TagExternalStorageTree)) - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetLatestVersionOfObject, err, zap.String("path", path)) return } if foundOID.IsDeleteMarker { - log.Error(logs.ObjectWasDeleted, logs.TagField(logs.TagExternalStorageTree)) - ResponseError(c, "object deleted", fasthttp.StatusNotFound) + h.logAndSendError(ctx, req, logs.ObjectWasDeleted, ErrObjectNotFound) return } addr := newAddress(cnrID, foundOID.OID) - handler(ctx, newRequest(c, log), addr) + handler(ctx, req, addr) } // byAttribute is a wrapper similar to byNativeAddress. -func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Context, request, oid.Address)) { - cidParam, _ := c.UserValue("cid").(string) - key, _ := c.UserValue("attr_key").(string) - val, _ := c.UserValue("attr_val").(string) - - ctx := utils.GetContextFromRequest(c) - log := utils.GetReqLogOrDefault(ctx, h.log) +func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { + cidParam, _ := req.UserValue("cid").(string) + key, _ := req.UserValue("attr_key").(string) + val, _ := req.UserValue("attr_val").(string) key, err := url.QueryUnescape(key) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), - zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToUnescapeQuery, err, zap.String("cid", cidParam), zap.String("attr_key", key)) return } val, err = url.QueryUnescape(val) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), - zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToUnescapeQuery, err, zap.String("cid", cidParam), zap.String("attr_val", key)) return } val = prepareAtribute(key, val) - log = log.With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val)) + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", cidParam), + zap.String("attr_key", key), zap.String("attr_val", val))) - bktInfo, err := h.getBucketInfo(ctx, cidParam, log) + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } - objID, err := h.findObjectByAttribute(ctx, log, bktInfo.CID, key, val) + objID, err := h.findObjectByAttribute(ctx, bktInfo.CID, key, val) if err != nil { if errors.Is(err, io.EOF) { - ResponseError(c, err.Error(), fasthttp.StatusNotFound) - return + err = fmt.Errorf("%w: %s", ErrObjectNotFound, err.Error()) } - - ResponseError(c, err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToFindObjectByAttribute, err) return } @@ -286,14 +276,13 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte addr.SetContainer(bktInfo.CID) addr.SetObject(objID) - handler(ctx, newRequest(c, log), addr) + handler(ctx, req, addr) } -func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { +func (h *Handler) findObjectByAttribute(ctx context.Context, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { res, err := h.search(ctx, cnrID, attrKey, attrVal, object.MatchStringEqual) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage)) - return oid.ID{}, fmt.Errorf("could not search for objects: %w", err) + return oid.ID{}, fmt.Errorf("search objects: %w", err) } defer res.Close() @@ -303,14 +292,14 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cn if n == 0 { switch { case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): - log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) + h.reqLogger(ctx).Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) attrVal = prepareAtribute(attrFileName, attrVal) - return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) + return h.findObjectByAttribute(ctx, cnrID, attrFileName, attrVal) case errors.Is(err, io.EOF): - log.Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) + h.reqLogger(ctx).Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) default: - log.Error(logs.ReadObjectListFailed, zap.Error(err), logs.TagField(logs.TagExternalStorage)) + h.reqLogger(ctx).Error(logs.ReadObjectListFailed, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("read object list failed: %w", err) } } @@ -369,13 +358,13 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci zone := h.config.FormContainerZone(namespace) cnrID, err = h.containerResolver.Resolve(ctx, zone, containerID) if err != nil && strings.Contains(err.Error(), "not found") { - err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error()) + err = fmt.Errorf("%w: %s", ErrContainerNotFound, err.Error()) } } return cnrID, err } -func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log *zap.Logger) (*data.BucketInfo, error) { +func (h *Handler) getBucketInfo(ctx context.Context, containerName string) (*data.BucketInfo, error) { ns, err := middleware.GetNamespace(ctx) if err != nil { return nil, err @@ -387,21 +376,16 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log * cnrID, err := h.resolveContainer(ctx, containerName) if err != nil { - log.Error(logs.CouldNotResolveContainerID, zap.Error(err), zap.String("cnrName", containerName), - logs.TagField(logs.TagDatapath)) - return nil, err + return nil, fmt.Errorf("resolve container: %w", err) } bktInfo, err := h.readContainer(ctx, *cnrID) if err != nil { - log.Error(logs.CouldNotGetContainerInfo, zap.Error(err), zap.String("cnrName", containerName), - zap.String("cnrName", cnrID.String()), - logs.TagField(logs.TagExternalStorage)) - return nil, err + return nil, fmt.Errorf("read container: %w", err) } if err = h.cache.Put(bktInfo); err != nil { - log.Warn(logs.CouldntPutBucketIntoCache, + h.reqLogger(ctx).Warn(logs.CouldntPutBucketIntoCache, zap.String("bucket name", bktInfo.Name), zap.Stringer("bucket cid", bktInfo.CID), zap.Error(err), @@ -434,31 +418,24 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } -func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.browseIndex") +func (h *Handler) browseIndex(ctx context.Context, req *fasthttp.RequestCtx, cidParam, oidParam string, isNativeList bool) { + ctx, span := tracing.StartSpanFromContext(ctx, "handler.browseIndex") defer span.End() - utils.SetContextToRequest(ctx, c) if !h.config.IndexPageEnabled() { - c.SetStatusCode(fasthttp.StatusNotFound) + req.SetStatusCode(fasthttp.StatusNotFound) return } - cidURLParam := c.UserValue("cid").(string) - oidURLParam := c.UserValue("oid").(string) - - reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With(zap.String("cid", cidURLParam), zap.String("oid", oidURLParam)) - - unescapedKey, err := url.QueryUnescape(oidURLParam) + unescapedKey, err := url.QueryUnescape(oidParam) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToUnescapeOIDParam, err) return } - bktInfo, err := h.getBucketInfo(ctx, cidURLParam, log) + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } @@ -468,7 +445,7 @@ func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) { listFunc = h.getDirObjectsNative } - h.browseObjects(c, browseParams{ + h.browseObjects(ctx, req, browseParams{ bucketInfo: bktInfo, prefix: unescapedKey, listObjects: listFunc, diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 3a81c50..93cb1d9 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -409,7 +409,7 @@ func TestFindObjectByAttribute(t *testing.T) { obj.SetAttributes(tc.firstAttr, tc.secondAttr) hc.cfg.additionalSearch = tc.additionalSearch - objID, err := hc.Handler().findObjectByAttribute(ctx, hc.Handler().log, cnrID, tc.reqAttrKey, tc.reqAttrValue) + objID, err := hc.Handler().findObjectByAttribute(ctx, cnrID, tc.reqAttrKey, tc.reqAttrValue) if tc.err != "" { require.Error(t, err) require.Contains(t, err.Error(), tc.err) diff --git a/internal/handler/head.go b/internal/handler/head.go index 7718c9c..11d45fc 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -27,7 +27,7 @@ const ( hdrContainerID = "X-Container-Id" ) -func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid.Address) { +func (h *Handler) headObject(ctx context.Context, req *fasthttp.RequestCtx, objectAddress oid.Address) { var start = time.Now() btoken := bearerToken(ctx) @@ -41,7 +41,7 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid obj, err := h.frostfs.HeadObject(ctx, prm) if err != nil { - req.handleFrostFSErr(err, start) + h.logAndSendError(ctx, req, logs.FailedToHeadObject, err, zap.Stringer("elapsed", time.Since(start))) return } @@ -65,7 +65,7 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid case object.AttributeTimestamp: value, err := strconv.ParseInt(val, 10, 64) if err != nil { - req.log.Info(logs.CouldntParseCreationDate, + h.reqLogger(ctx).Info(logs.CouldntParseCreationDate, zap.String("key", key), zap.String("val", val), zap.Error(err), @@ -100,7 +100,7 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid return h.frostfs.RangeObject(ctx, prmRange) }, filename) if err != nil && err != io.EOF { - req.handleFrostFSErr(err, start) + h.logAndSendError(ctx, req, logs.FailedToDetectContentTypeFromPayload, err, zap.Stringer("elapsed", time.Since(start))) return } } @@ -116,48 +116,44 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) { } // HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format. -func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByAddressOrBucketName") +func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.HeadByAddressOrBucketName") defer span.End() - cidParam, _ := c.UserValue("cid").(string) - oidParam, _ := c.UserValue("oid").(string) + cidParam, _ := req.UserValue("cid").(string) + oidParam, _ := req.UserValue("oid").(string) - log := utils.GetReqLogOrDefault(ctx, h.log).With( + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( zap.String("cid", cidParam), zap.String("oid", oidParam), - ) + )) - bktInfo, err := h.getBucketInfo(ctx, cidParam, log) + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } + checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { - log.Error(logs.FailedToCheckIfSettingsNodeExist, zap.String("cid", bktInfo.CID.String()), - zap.Error(checkS3Err), logs.TagField(logs.TagExternalStorageTree)) - logAndSendBucketError(c, log, checkS3Err) + h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) return } - req := newRequest(c, log) - var objID oid.ID if checkS3Err == nil { h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject) } else if err = objID.DecodeString(oidParam); err == nil { h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { - logAndSendBucketError(c, log, checkS3Err) + h.logAndSendError(ctx, req, logs.InvalidOIDParam, err) } } // HeadByAttribute handles attribute-based head requests. -func (h *Handler) HeadByAttribute(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByAttribute") +func (h *Handler) HeadByAttribute(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.HeadByAttribute") defer span.End() - utils.SetContextToRequest(ctx, c) - h.byAttribute(c, h.headObject) + h.byAttribute(ctx, req, h.headObject) } diff --git a/internal/handler/multipart.go b/internal/handler/multipart.go index 5ed2350..5b06882 100644 --- a/internal/handler/multipart.go +++ b/internal/handler/multipart.go @@ -1,6 +1,7 @@ package handler import ( + "context" "errors" "io" "strconv" @@ -53,7 +54,7 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF } // getPayload returns initial payload if object is not multipart else composes new reader with parts data. -func (h *Handler) getPayload(p getMultiobjectBodyParams) (io.ReadCloser, uint64, error) { +func (h *Handler) getPayload(ctx context.Context, p getMultiobjectBodyParams) (io.ReadCloser, uint64, error) { cid, ok := p.obj.Header.ContainerID() if !ok { return nil, 0, errors.New("no container id set") @@ -66,7 +67,6 @@ func (h *Handler) getPayload(p getMultiobjectBodyParams) (io.ReadCloser, uint64, if err != nil { return nil, 0, err } - ctx := p.req.RequestCtx params := PrmInitMultiObjectReader{ Addr: newAddress(cid, oid), Bearer: bearerToken(ctx), diff --git a/internal/handler/reader.go b/internal/handler/reader.go index e8ac098..711bfd2 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -63,11 +63,10 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error), file type getMultiobjectBodyParams struct { obj *Object - req request strSize string } -func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.Address) { +func (h *Handler) receiveFile(ctx context.Context, req *fasthttp.RequestCtx, objAddress oid.Address) { var ( shouldDownload = req.QueryArgs().GetBool("download") start = time.Now() @@ -85,12 +84,12 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A rObj, err := h.frostfs.GetObject(ctx, prm) if err != nil { - req.handleFrostFSErr(err, start) + h.logAndSendError(ctx, req, logs.FailedToGetObject, err, zap.Stringer("elapsed", time.Since(start))) return } // we can't close reader in this function, so how to do it? - req.setIDs(rObj.Header) + setIDs(req, rObj.Header) payload := rObj.Payload payloadSize := rObj.Header.PayloadSize() for _, attr := range rObj.Header.Attributes() { @@ -107,8 +106,8 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A case object.AttributeFileName: filename = val case object.AttributeTimestamp: - if err = req.setTimestamp(val); err != nil { - req.log.Error(logs.CouldntParseCreationDate, + if err = setTimestamp(req, val); err != nil { + h.reqLogger(ctx).Error(logs.CouldntParseCreationDate, zap.String("val", val), zap.Error(err), logs.TagField(logs.TagDatapath)) @@ -118,13 +117,12 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A case object.AttributeFilePath: filepath = val case attributeMultipartObjectSize: - payload, payloadSize, err = h.getPayload(getMultiobjectBodyParams{ + payload, payloadSize, err = h.getPayload(ctx, getMultiobjectBodyParams{ obj: rObj, - req: req, strSize: val, }) if err != nil { - req.handleFrostFSErr(err, start) + h.logAndSendError(ctx, req, logs.FailedToGetObjectPayload, err, zap.Stringer("elapsed", time.Since(start))) return } } @@ -133,7 +131,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A filename = filepath } - req.setDisposition(shouldDownload, filename) + setDisposition(req, shouldDownload, filename) req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) @@ -145,8 +143,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A return payload, nil }, filename) if err != nil && err != io.EOF { - req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToDetectContentTypeFromPayload, err, zap.Stringer("elapsed", time.Since(start))) return } @@ -165,7 +162,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A req.Response.SetBodyStream(payload, int(payloadSize)) } -func (r *request) setIDs(obj object.Object) { +func setIDs(r *fasthttp.RequestCtx, obj object.Object) { objID, _ := obj.ID() cnrID, _ := obj.ContainerID() r.Response.Header.Set(hdrObjectID, objID.String()) @@ -173,7 +170,7 @@ func (r *request) setIDs(obj object.Object) { r.Response.Header.Set(hdrContainerID, cnrID.String()) } -func (r *request) setDisposition(shouldDownload bool, filename string) { +func setDisposition(r *fasthttp.RequestCtx, shouldDownload bool, filename string) { const ( inlineDisposition = "inline" attachmentDisposition = "attachment" @@ -187,7 +184,7 @@ func (r *request) setDisposition(shouldDownload bool, filename string) { r.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) } -func (r *request) setTimestamp(timestamp string) error { +func setTimestamp(r *fasthttp.RequestCtx, timestamp string) error { value, err := strconv.ParseInt(timestamp, 10, 64) if err != nil { return err diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 48d0495..05f4c97 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -50,44 +50,41 @@ func (pr *putResponse) encode(w io.Writer) error { } // Upload handles multipart upload request. -func (h *Handler) Upload(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.Upload") +func (h *Handler) Upload(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.Upload") defer span.End() - utils.SetContextToRequest(ctx, c) var file MultipartFile - scid, _ := c.UserValue("cid").(string) - bodyStream := c.RequestBodyStream() + scid, _ := req.UserValue("cid").(string) + bodyStream := req.RequestBodyStream() drainBuf := make([]byte, drainBufSize) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With(zap.String("cid", scid)) + log := h.reqLogger(ctx) + ctx = utils.SetReqLog(ctx, log.With(zap.String("cid", scid))) - bktInfo, err := h.getBucketInfo(ctx, scid, log) + bktInfo, err := h.getBucketInfo(ctx, scid) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } - boundary := string(c.Request.Header.MultipartFormBoundary()) + boundary := string(req.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { - log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.CouldNotReceiveMultipartForm, err) return } - filtered, err := filterHeaders(log, &c.Request.Header) + filtered, err := filterHeaders(log, &req.Request.Header) if err != nil { - log.Error(logs.FailedToFilterHeaders, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToFilterHeaders, err) return } - if c.Request.Header.Peek(explodeArchiveHeader) != nil { - h.explodeArchive(request{c, log}, bktInfo, file, filtered) + if req.Request.Header.Peek(explodeArchiveHeader) != nil { + h.explodeArchive(ctx, req, bktInfo, file, filtered) } else { - h.uploadSingleObject(request{c, log}, bktInfo, file, filtered) + h.uploadSingleObject(ctx, req, bktInfo, file, filtered) } // Multipart is multipart and thus can contain more than one part which @@ -104,46 +101,39 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { } } -func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file MultipartFile, filtered map[string]string) { - c, log := req.RequestCtx, req.log - - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.uploadSingleObject") +func (h *Handler) uploadSingleObject(ctx context.Context, req *fasthttp.RequestCtx, bkt *data.BucketInfo, file MultipartFile, filtered map[string]string) { + ctx, span := tracing.StartSpanFromContext(ctx, "handler.uploadSingleObject") defer span.End() - utils.SetContextToRequest(ctx, c) setIfNotExist(filtered, object.AttributeFileName, file.FileName()) - attributes, err := h.extractAttributes(c, log, filtered) + attributes, err := h.extractAttributes(ctx, req, filtered) if err != nil { - log.Error(logs.FailedToGetAttributes, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToGetAttributes, err) return } - idObj, err := h.uploadObject(c, bkt, attributes, file) + idObj, err := h.uploadObject(ctx, bkt, attributes, file) if err != nil { - h.handlePutFrostFSErr(c, err, log) + h.logAndSendError(ctx, req, logs.FailedToUploadObject, err) return } - log.Debug(logs.ObjectUploaded, + h.reqLogger(ctx).Debug(logs.ObjectUploaded, zap.String("oid", idObj.EncodeToString()), zap.String("FileName", file.FileName()), logs.TagField(logs.TagExternalStorage), ) addr := newAddress(bkt.CID, idObj) - c.Response.Header.SetContentType(jsonHeader) + req.Response.Header.SetContentType(jsonHeader) // Try to return the response, otherwise, if something went wrong, throw an error. - if err = newPutResponse(addr).encode(c); err != nil { - log.Error(logs.CouldNotEncodeResponse, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) + if err = newPutResponse(addr).encode(req); err != nil { + h.logAndSendError(ctx, req, logs.CouldNotEncodeResponse, err) return } } -func (h *Handler) uploadObject(c *fasthttp.RequestCtx, bkt *data.BucketInfo, attrs []object.Attribute, file io.Reader) (oid.ID, error) { - ctx := utils.GetContextFromRequest(c) - +func (h *Handler) uploadObject(ctx context.Context, bkt *data.BucketInfo, attrs []object.Attribute, file io.Reader) (oid.ID, error) { obj := object.New() obj.SetContainerID(bkt.CID) obj.SetOwnerID(*h.ownerID) @@ -168,19 +158,18 @@ func (h *Handler) uploadObject(c *fasthttp.RequestCtx, bkt *data.BucketInfo, att return idObj, nil } -func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, filtered map[string]string) ([]object.Attribute, error) { - ctx := utils.GetContextFromRequest(c) +func (h *Handler) extractAttributes(ctx context.Context, req *fasthttp.RequestCtx, filtered map[string]string) ([]object.Attribute, error) { now := time.Now() - if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { + if rawHeader := req.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { - log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err), + h.reqLogger(ctx).Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err), logs.TagField(logs.TagDatapath)) } else { now = parsed } } if err := utils.PrepareExpirationHeader(ctx, h.frostfs, filtered, now); err != nil { - log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err), logs.TagField(logs.TagDatapath)) return nil, err } attributes := make([]object.Attribute, 0, len(filtered)) @@ -207,38 +196,33 @@ func newAttribute(key string, val string) object.Attribute { // explodeArchive read files from archive and creates objects for each of them. // Sets FilePath attribute with name from tar.Header. -func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.ReadCloser, filtered map[string]string) { - c, log := req.RequestCtx, req.log - - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.explodeArchive") +func (h *Handler) explodeArchive(ctx context.Context, req *fasthttp.RequestCtx, bkt *data.BucketInfo, file io.ReadCloser, filtered map[string]string) { + ctx, span := tracing.StartSpanFromContext(ctx, "handler.explodeArchive") defer span.End() - utils.SetContextToRequest(ctx, c) // remove user attributes which vary for each file in archive // to guarantee that they won't appear twice delete(filtered, object.AttributeFileName) delete(filtered, object.AttributeFilePath) - commonAttributes, err := h.extractAttributes(c, log, filtered) + commonAttributes, err := h.extractAttributes(ctx, req, filtered) if err != nil { - log.Error(logs.FailedToGetAttributes, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToGetAttributes, err) return } attributes := commonAttributes reader := file - if bytes.EqualFold(c.Request.Header.Peek(fasthttp.HeaderContentEncoding), []byte("gzip")) { - log.Debug(logs.GzipReaderSelected, logs.TagField(logs.TagDatapath)) + if bytes.EqualFold(req.Request.Header.Peek(fasthttp.HeaderContentEncoding), []byte("gzip")) { + h.reqLogger(ctx).Debug(logs.GzipReaderSelected, logs.TagField(logs.TagDatapath)) gzipReader, err := gzip.NewReader(file) if err != nil { - log.Error(logs.FailedToCreateGzipReader, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could read gzip file: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToCreateGzipReader, err) return } defer func() { if err := gzipReader.Close(); err != nil { - log.Warn(logs.FailedToCloseReader, zap.Error(err), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Warn(logs.FailedToCloseReader, zap.Error(err), logs.TagField(logs.TagDatapath)) } }() reader = gzipReader @@ -250,8 +234,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read if errors.Is(err, io.EOF) { break } else if err != nil { - log.Error(logs.FailedToReadFileFromTar, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not get next entry: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToReadFileFromTar, err) return } @@ -265,13 +248,13 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read attributes = append(attributes, newAttribute(object.AttributeFilePath, obj.Name)) attributes = append(attributes, newAttribute(object.AttributeFileName, fileName)) - idObj, err := h.uploadObject(c, bkt, attributes, tarReader) + idObj, err := h.uploadObject(ctx, bkt, attributes, tarReader) if err != nil { - h.handlePutFrostFSErr(c, err, log) + h.logAndSendError(ctx, req, logs.FailedToUploadObject, err) return } - log.Debug(logs.ObjectUploaded, + h.reqLogger(ctx).Debug(logs.ObjectUploaded, zap.String("oid", idObj.EncodeToString()), zap.String("FileName", fileName), logs.TagField(logs.TagExternalStorage), @@ -279,14 +262,6 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read } } -func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *zap.Logger) { - statusCode, msg, additionalFields := formErrorResponse("could not store file in frostfs", err) - logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) - - log.Error(logs.CouldNotStoreFileInFrostfs, append(logFields, logs.TagField(logs.TagExternalStorage))...) - ResponseError(r, msg, statusCode) -} - func (h *Handler) fetchBearerToken(ctx context.Context) *bearer.Token { if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { return tkn diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 0a1dc62..8cb070d 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -5,13 +5,12 @@ import ( "errors" "fmt" "strings" - "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -19,30 +18,6 @@ import ( "go.uber.org/zap" ) -type request struct { - *fasthttp.RequestCtx - log *zap.Logger -} - -func newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) request { - return request{ - RequestCtx: ctx, - log: log, - } -} - -func (r *request) handleFrostFSErr(err error, start time.Time) { - logFields := []zap.Field{ - zap.Stringer("elapsed", time.Since(start)), - zap.Error(err), - } - statusCode, msg, additionalFields := formErrorResponse("could not receive object", err) - logFields = append(logFields, additionalFields...) - - r.log.Error(logs.CouldNotReceiveObject, append(logFields, logs.TagField(logs.TagExternalStorage))...) - ResponseError(r.RequestCtx, msg, statusCode) -} - func bearerToken(ctx context.Context) *bearer.Token { if tkn, err := tokens.LoadBearerToken(ctx); err == nil { return tkn @@ -84,14 +59,16 @@ func isValidValue(s string) bool { return true } -func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { - log.Error(logs.CouldNotGetBucket, zap.Error(err), logs.TagField(logs.TagDatapath)) +func (h *Handler) reqLogger(ctx context.Context) *zap.Logger { + return utils.GetReqLogOrDefault(ctx, h.log) +} - if client.IsErrContainerNotFound(err) { - ResponseError(c, "Not Found", fasthttp.StatusNotFound) - return - } - ResponseError(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) +func (h *Handler) logAndSendError(ctx context.Context, c *fasthttp.RequestCtx, msg string, err error, additional ...zap.Field) { + utils.GetReqLogOrDefault(ctx, h.log).Error(msg, + append([]zap.Field{zap.Error(err), logs.TagField(logs.TagDatapath)}, additional...)...) + + msg, code := formErrorResponse(err) + ResponseError(c, msg, code) } func newAddress(cnr cid.ID, obj oid.ID) oid.Address { @@ -112,31 +89,23 @@ func ResponseError(r *fasthttp.RequestCtx, msg string, code int) { r.Error(msg+"\n", code) } -func formErrorResponse(message string, err error) (int, string, []zap.Field) { - var ( - msg string - statusCode int - logFields []zap.Field - ) - - st := new(sdkstatus.ObjectAccessDenied) - +func formErrorResponse(err error) (string, int) { switch { - case errors.As(err, &st): - statusCode = fasthttp.StatusForbidden - reason := st.Reason() - msg = fmt.Sprintf("%s: %v: %s", message, err, reason) - logFields = append(logFields, zap.String("error_detail", reason)) + case errors.Is(err, ErrAccessDenied): + return fmt.Sprintf("Storage Access Denied:\n%v", err), fasthttp.StatusForbidden + case errors.Is(err, layer.ErrNodeAccessDenied): + return fmt.Sprintf("Tree Access Denied:\n%v", err), fasthttp.StatusForbidden case errors.Is(err, ErrQuotaLimitReached): - statusCode = fasthttp.StatusConflict - msg = fmt.Sprintf("%s: %v", message, err) - case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err): - statusCode = fasthttp.StatusNotFound - msg = "Not Found" + return fmt.Sprintf("Quota Reached:\n%v", err), fasthttp.StatusConflict + case errors.Is(err, ErrContainerNotFound): + return fmt.Sprintf("Container Not Found:\n%v", err), fasthttp.StatusNotFound + case errors.Is(err, ErrObjectNotFound): + return fmt.Sprintf("Object Not Found:\n%v", err), fasthttp.StatusNotFound + case errors.Is(err, layer.ErrNodeNotFound): + return fmt.Sprintf("Tree Node Not Found:\n%v", err), fasthttp.StatusNotFound + case errors.Is(err, ErrGatewayTimeout): + return fmt.Sprintf("Gateway Timeout:\n%v", err), fasthttp.StatusGatewayTimeout default: - statusCode = fasthttp.StatusBadRequest - msg = fmt.Sprintf("%s: %v", message, err) + return fmt.Sprintf("Bad Request:\n%v", err), fasthttp.StatusBadRequest } - - return statusCode, msg, logFields } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 3166f98..3e9b931 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -78,7 +78,7 @@ const ( // Log messages with the "datapath" tag. const ( CouldntParseCreationDate = "couldn't parse creation date" - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" + FailedToDetectContentTypeFromPayload = "failed to detect Content-Type from payload" FailedToAddObjectToArchive = "failed to add object to archive" CloseZipWriter = "close zip writer" IgnorePartEmptyFormName = "ignore part, empty form name" @@ -105,9 +105,21 @@ const ( CouldNotReceiveMultipartForm = "could not receive multipart/form" ObjectsNotFound = "objects not found" IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" - CouldNotGetBucket = "could not get bucket" - CouldNotResolveContainerID = "could not resolve container id" - FailedToSumbitTaskToPool = "failed to submit task to pool" + FailedToGetBucketInfo = "could not get bucket info" + FailedToSubmitTaskToPool = "failed to submit task to pool" + ObjectWasDeleted = "object was deleted" + FailedToGetLatestVersionOfObject = "failed to get latest version of object" + FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists" + FailedToListObjects = "failed to list objects" + FailedToParseTemplate = "failed to parse template" + FailedToExecuteTemplate = "failed to execute template" + FailedToUploadObject = "failed to upload object" + FailedToHeadObject = "failed to head object" + FailedToGetObject = "failed to get object" + FailedToGetObjectPayload = "failed to get object payload" + FailedToFindObjectByAttribute = "failed to get find object by attribute" + FailedToUnescapeOIDParam = "failed to unescape oid param" + InvalidOIDParam = "invalid oid param" CouldNotGetCORSConfiguration = "could not get cors configuration" EmptyOriginRequestHeader = "empty Origin request header" EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header" @@ -117,21 +129,13 @@ const ( // Log messages with the "external_storage" tag. const ( - CouldNotReceiveObject = "could not receive object" - CouldNotSearchForObjects = "could not search for objects" ObjectNotFound = "object not found" ReadObjectListFailed = "read object list failed" - CouldNotStoreFileInFrostfs = "could not store file in frostfs" - FailedToHeadObject = "failed to head object" ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" - FailedToGetObject = "failed to get object" ObjectUploaded = "object uploaded" - CouldNotGetContainerInfo = "could not get container info" ) // Log messages with the "external_storage_tree" tag. const ( - ObjectWasDeleted = "object was deleted" - FailedToGetLatestVersionOfObject = "failed to get latest version of object" - FailedToCheckIfSettingsNodeExist = "Failed to check if settings node exists" + FoundSeveralSystemTreeNodes = "found several system tree nodes" ) diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index 4cf45a4..1841446 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -45,7 +46,7 @@ func (x *FrostFS) Container(ctx context.Context, containerPrm handler.PrmContain res, err := x.pool.GetContainer(ctx, prm) if err != nil { - return nil, handleObjectError("read container via connection pool", err) + return nil, handleStorageError("read container via connection pool", err) } return &res, nil @@ -69,7 +70,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) idObj, err := x.pool.PutObject(ctx, prmPut) if err != nil { - return oid.ID{}, handleObjectError("save object via connection pool", err) + return oid.ID{}, handleStorageError("save object via connection pool", err) } return idObj.ObjectID, nil } @@ -85,7 +86,7 @@ func (x payloadReader) Read(p []byte) (int, error) { if err != nil && errors.Is(err, io.EOF) { return n, err } - return n, handleObjectError("read payload", err) + return n, handleStorageError("read payload", err) } // HeadObject implements frostfs.FrostFS interface method. @@ -102,7 +103,7 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o res, err := x.pool.HeadObject(ctx, prmHead) if err != nil { - return nil, handleObjectError("read object header via connection pool", err) + return nil, handleStorageError("read object header via connection pool", err) } return &res, nil @@ -122,7 +123,7 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han res, err := x.pool.GetObject(ctx, prmGet) if err != nil { - return nil, handleObjectError("init full object reading via connection pool", err) + return nil, handleStorageError("init full object reading via connection pool", err) } return &handler.Object{ @@ -147,7 +148,7 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) ( res, err := x.pool.ObjectRange(ctx, prmRange) if err != nil { - return nil, handleObjectError("init payload range reading via connection pool", err) + return nil, handleStorageError("init payload range reading via connection pool", err) } return payloadReader{&res}, nil @@ -168,7 +169,7 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch res, err := x.pool.SearchObjects(ctx, prmSearch) if err != nil { - return nil, handleObjectError("init object search via connection pool", err) + return nil, handleStorageError("init object search via connection pool", err) } return &res, nil @@ -202,7 +203,7 @@ func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) { netmapSnapshot, err := x.pool.NetMapSnapshot(ctx) if err != nil { - return netmapSnapshot, handleObjectError("get netmap via connection pool", err) + return netmapSnapshot, handleStorageError("get netmap via connection pool", err) } return netmapSnapshot, nil @@ -226,7 +227,7 @@ func (x *ResolverFrostFS) SystemDNS(ctx context.Context) (string, error) { networkInfo, err := x.pool.NetworkInfo(ctx) if err != nil { - return "", handleObjectError("read network info via client", err) + return "", handleStorageError("read network info via client", err) } domain := networkInfo.RawNetworkParameter("SystemDNS") @@ -237,7 +238,7 @@ func (x *ResolverFrostFS) SystemDNS(ctx context.Context) (string, error) { return string(domain), nil } -func handleObjectError(msg string, err error) error { +func handleStorageError(msg string, err error) error { if err == nil { return nil } @@ -250,6 +251,14 @@ func handleObjectError(msg string, err error) error { return fmt.Errorf("%s: %w: %s", msg, handler.ErrAccessDenied, reason) } + if client.IsErrContainerNotFound(err) { + return fmt.Errorf("%s: %w: %s", msg, handler.ErrContainerNotFound, err.Error()) + } + + if client.IsErrObjectNotFound(err) { + return fmt.Errorf("%s: %w: %s", msg, handler.ErrObjectNotFound, err.Error()) + } + if IsTimeoutError(err) { return fmt.Errorf("%s: %w: %s", msg, handler.ErrGatewayTimeout, err.Error()) } diff --git a/internal/service/frostfs/frostfs_test.go b/internal/service/frostfs/frostfs_test.go index e9b3329..e4344f7 100644 --- a/internal/service/frostfs/frostfs_test.go +++ b/internal/service/frostfs/frostfs_test.go @@ -18,7 +18,7 @@ func TestHandleObjectError(t *testing.T) { msg := "some msg" t.Run("nil error", func(t *testing.T) { - err := handleObjectError(msg, nil) + err := handleStorageError(msg, nil) require.Nil(t, err) }) @@ -27,7 +27,7 @@ func TestHandleObjectError(t *testing.T) { inputErr := new(apistatus.ObjectAccessDenied) inputErr.WriteReason(reason) - err := handleObjectError(msg, inputErr) + err := handleStorageError(msg, inputErr) require.ErrorIs(t, err, handler.ErrAccessDenied) require.Contains(t, err.Error(), reason) require.Contains(t, err.Error(), msg) @@ -38,7 +38,7 @@ func TestHandleObjectError(t *testing.T) { inputErr := new(apistatus.ObjectAccessDenied) inputErr.WriteReason(reason) - err := handleObjectError(msg, inputErr) + err := handleStorageError(msg, inputErr) require.ErrorIs(t, err, handler.ErrQuotaLimitReached) require.Contains(t, err.Error(), reason) require.Contains(t, err.Error(), msg) @@ -47,7 +47,7 @@ func TestHandleObjectError(t *testing.T) { t.Run("simple timeout", func(t *testing.T) { inputErr := errors.New("timeout") - err := handleObjectError(msg, inputErr) + err := handleStorageError(msg, inputErr) require.ErrorIs(t, err, handler.ErrGatewayTimeout) require.Contains(t, err.Error(), inputErr.Error()) require.Contains(t, err.Error(), msg) @@ -58,7 +58,7 @@ func TestHandleObjectError(t *testing.T) { defer cancel() <-ctx.Done() - err := handleObjectError(msg, ctx.Err()) + err := handleStorageError(msg, ctx.Err()) require.ErrorIs(t, err, handler.ErrGatewayTimeout) require.Contains(t, err.Error(), ctx.Err().Error()) require.Contains(t, err.Error(), msg) @@ -67,7 +67,7 @@ func TestHandleObjectError(t *testing.T) { t.Run("grpc deadline exceeded", func(t *testing.T) { inputErr := fmt.Errorf("wrap grpc error: %w", status.Error(codes.DeadlineExceeded, "error")) - err := handleObjectError(msg, inputErr) + err := handleStorageError(msg, inputErr) require.ErrorIs(t, err, handler.ErrGatewayTimeout) require.Contains(t, err.Error(), inputErr.Error()) require.Contains(t, err.Error(), msg) @@ -76,7 +76,7 @@ func TestHandleObjectError(t *testing.T) { t.Run("unknown error", func(t *testing.T) { inputErr := errors.New("unknown error") - err := handleObjectError(msg, inputErr) + err := handleStorageError(msg, inputErr) require.ErrorIs(t, err, inputErr) require.Contains(t, err.Error(), msg) }) diff --git a/internal/service/frostfs/tree_pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go index 410acda..d0b5501 100644 --- a/internal/service/frostfs/tree_pool_wrapper.go +++ b/internal/service/frostfs/tree_pool_wrapper.go @@ -63,7 +63,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([ nodes, err := w.p.GetNodes(ctx, poolPrm) if err != nil { - return nil, handleError(err) + return nil, handleTreeError(err) } res := make([]tree.NodeResponse, len(nodes)) @@ -82,7 +82,7 @@ func getBearer(ctx context.Context) []byte { return token.Marshal() } -func handleError(err error) error { +func handleTreeError(err error) error { if err == nil { return nil } @@ -122,7 +122,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) if err != nil { - return nil, handleError(err) + return nil, handleTreeError(err) } var subtree []tree.NodeResponse @@ -133,7 +133,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, node, err = subTreeReader.Next() } if err != io.EOF { - return nil, handleError(err) + return nil, handleTreeError(err) } return subtree, nil diff --git a/tree/tree.go b/tree/tree.go index 315e5ad..2ee9356 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -8,14 +8,18 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" ) type ( Tree struct { service ServiceClient + log *zap.Logger } // ServiceClient is a client to interact with tree service. @@ -73,8 +77,8 @@ const ( ) // NewTree creates instance of Tree using provided address and create grpc connection. -func NewTree(service ServiceClient) *Tree { - return &Tree{service: service} +func NewTree(service ServiceClient, log *zap.Logger) *Tree { + return &Tree{service: service, log: log} } type Meta interface { @@ -257,6 +261,9 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name if len(nodes) == 0 { return nil, layer.ErrNodeNotFound } + if len(nodes) != 1 { + c.reqLogger(ctx).Warn(logs.FoundSeveralSystemTreeNodes, zap.String("name", name), logs.TagField(logs.TagExternalStorageTree)) + } return newMultiNode(nodes) } @@ -296,7 +303,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { } if targetIndexNode == -1 { - return nil, layer.ErrNodeNotFound + return nil, fmt.Errorf("latest version: %w", layer.ErrNodeNotFound) } return nodes[targetIndexNode], nil @@ -423,6 +430,10 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr return intermediateNodes, nil } +func (c *Tree) reqLogger(ctx context.Context) *zap.Logger { + return utils.GetReqLogOrDefault(ctx, c.log) +} + func GetFilename(node NodeResponse) string { for _, kv := range node.GetMeta() { if kv.GetKey() == FileNameKey { From f0b86c8ba7e861c8a482cf60ba32b837dee18efb Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 4 Mar 2025 14:22:10 +0300 Subject: [PATCH 75/86] [#191] Update integration tests Signed-off-by: Denis Kirillov --- cmd/http-gw/integration_test.go | 119 ++++++++++++++++++++++++++++---- 1 file changed, 104 insertions(+), 15 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 20b4c8b..6ab8e99 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -20,9 +20,11 @@ import ( containerv2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -99,6 +101,7 @@ func TestIntegration(t *testing.T) { t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID) }) t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID) }) t.Run("test namespaces "+version, func(t *testing.T) { checkNamespaces(ctx, t, clientPool, ownerID, CID) }) + t.Run("test status codes "+version, func(t *testing.T) { checkStatusCodes(ctx, t, clientPool, ownerID, version) }) cancel() server.Wait() @@ -267,7 +270,7 @@ func putWithDuplicateKeys(t *testing.T, CID cid.ID) { body, err := io.ReadAll(resp.Body) require.NoError(t, err) - require.Equal(t, "key duplication error: "+attr+"\n", string(body)) + require.Contains(t, string(body), "key duplication error: "+attr+"\n") require.Equal(t, http.StatusBadRequest, resp.StatusCode) } @@ -436,7 +439,80 @@ func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, o resp, err = http.DefaultClient.Do(req) require.NoError(t, err) require.Equal(t, http.StatusNotFound, resp.StatusCode) +} +func checkStatusCodes(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, version string) { + cli := http.Client{Timeout: 30 * time.Second} + + t.Run("container not found by name", func(t *testing.T) { + resp, err := cli.Get(testHost + "/get/unknown/object") + require.NoError(t, err) + require.Equal(t, http.StatusNotFound, resp.StatusCode) + requireBodyContains(t, resp, "container not found") + }) + + t.Run("container not found by cid", func(t *testing.T) { + cnrIDTest := cidtest.ID() + resp, err := cli.Get(testHost + "/get/" + cnrIDTest.EncodeToString() + "/object") + require.NoError(t, err) + requireBodyContains(t, resp, "container not found") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + + t.Run("object not found in storage", func(t *testing.T) { + resp, err := cli.Get(testHost + "/get_by_attribute/" + testContainerName + "/FilePath/object2") + require.NoError(t, err) + requireBodyContains(t, resp, "object not found") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + + t.Run("access denied", func(t *testing.T) { + basicACL := acl.Private + var recs []*eacl.Record + if version == "1.2.7" { + basicACL = acl.PublicRWExtended + rec := eacl.NewRecord() + rec.SetAction(eacl.ActionDeny) + rec.SetOperation(eacl.OperationGet) + recs = append(recs, rec) + } + + cnrID, err := createContainerBase(ctx, t, clientPool, ownerID, basicACL, "") + require.NoError(t, err) + + key, err := keys.NewPrivateKey() + require.NoError(t, err) + jsonToken, _ := makeBearerTokens(t, key, ownerID, version, recs...) + + t.Run("get", func(t *testing.T) { + request, err := http.NewRequest(http.MethodGet, testHost+"/get/"+cnrID.EncodeToString()+"/object", nil) + require.NoError(t, err) + request.Header.Set("Authorization", "Bearer "+jsonToken) + + resp, err := cli.Do(request) + require.NoError(t, err) + requireBodyContains(t, resp, "access denied") + require.Equal(t, http.StatusForbidden, resp.StatusCode) + }) + + t.Run("upload", func(t *testing.T) { + request, _, _ := makePutRequest(t, testHost+"/upload/"+cnrID.EncodeToString()) + request.Header.Set("Authorization", "Bearer "+jsonToken) + + resp, err := cli.Do(request) + require.NoError(t, err) + requireBodyContains(t, resp, "access denied") + require.Equal(t, http.StatusForbidden, resp.StatusCode) + }) + }) +} + +func requireBodyContains(t *testing.T, resp *http.Response, msg string) { + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + defer resp.Body.Close() + + require.Contains(t, strings.ToLower(string(data)), strings.ToLower(msg)) } func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container { @@ -485,6 +561,10 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool } func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, name string) (cid.ID, error) { + return createContainerBase(ctx, t, clientPool, ownerID, acl.PublicRWExtended, name) +} + +func createContainerBase(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, basicACL acl.Basic, name string) (cid.ID, error) { var policy netmap.PlacementPolicy err := policy.DecodeString("REP 1") require.NoError(t, err) @@ -492,24 +572,28 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o var cnr container.Container cnr.Init() cnr.SetPlacementPolicy(policy) - cnr.SetBasicACL(acl.PublicRWExtended) + cnr.SetBasicACL(basicACL) cnr.SetOwner(ownerID) container.SetCreationTime(&cnr, time.Now()) - var domain container.Domain - domain.SetName(name) + if name != "" { + var domain container.Domain + domain.SetName(name) - cnr.SetAttribute(containerv2.SysAttributeName, domain.Name()) - cnr.SetAttribute(containerv2.SysAttributeZone, domain.Zone()) + cnr.SetAttribute(containerv2.SysAttributeName, domain.Name()) + cnr.SetAttribute(containerv2.SysAttributeZone, domain.Zone()) + } - var waitPrm pool.WaitParams - waitPrm.SetTimeout(15 * time.Second) - waitPrm.SetPollInterval(3 * time.Second) - - var prm pool.PrmContainerPut - prm.SetContainer(cnr) - prm.SetWaitParams(waitPrm) + prm := pool.PrmContainerPut{ + ClientParams: client.PrmContainerPut{ + Container: &cnr, + }, + WaitParams: &pool.WaitParams{ + Timeout: 15 * time.Second, + PollInterval: 3 * time.Second, + }, + } CID, err := clientPool.PutContainer(ctx, prm) if err != nil { @@ -556,13 +640,18 @@ func registerUser(t *testing.T, ctx context.Context, aioContainer testcontainers require.NoError(t, err) } -func makeBearerTokens(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) (jsonTokenBase64, binaryTokenBase64 string) { +func makeBearerTokens(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string, records ...*eacl.Record) (jsonTokenBase64, binaryTokenBase64 string) { tkn := new(bearer.Token) tkn.ForUser(ownerID) tkn.SetExp(10000) if version == "1.2.7" { - tkn.SetEACLTable(*eacl.NewTable()) + table := eacl.NewTable() + for i := range records { + table.AddRecord(records[i]) + } + + tkn.SetEACLTable(*table) } else { tkn.SetImpersonate(true) } From cb72d11515af5b8fbc602e8ca4e50578d68f7e3b Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Fri, 21 Mar 2025 13:38:43 +0300 Subject: [PATCH 76/86] [#224] Refactor logger tag configuration Signed-off-by: Pavel Pogodaev --- CHANGELOG.md | 1 + cmd/http-gw/app.go | 31 +++++++++++++++++++++++++++---- cmd/http-gw/logger.go | 19 +++++++++++-------- cmd/http-gw/settings.go | 13 +++++++++---- config/config.env | 5 +++-- config/config.yaml | 3 +-- docs/gate-configuration.md | 15 +++++++-------- 7 files changed, 59 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2025b6d..85798b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This document outlines major changes between releases. ### Added - Add handling quota limit reached error (#187) - Add slash clipping for FileName attribute (#174) +- Add new format of tag names config ## [0.32.3] - 2025-02-05 diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index de186fb..ca7797f 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -114,7 +114,8 @@ type ( } tagsConfig struct { - tagLogs sync.Map + tagLogs sync.Map + defaultLvl zap.AtomicLevel } logLevelConfig struct { @@ -134,19 +135,34 @@ func newLogLevel(v *viper.Viper) zap.AtomicLevel { } func newTagsConfig(v *viper.Viper, ll zapcore.Level) *tagsConfig { - var t tagsConfig + t := tagsConfig{defaultLvl: zap.NewAtomicLevelAt(ll)} if err := t.update(v, ll); err != nil { // panic here is analogue of the similar panic during common log level initialization. panic(err.Error()) } + return &t } func newLogLevelConfig(lvl zap.AtomicLevel, tagsConfig *tagsConfig) *logLevelConfig { - return &logLevelConfig{ + cfg := &logLevelConfig{ logLevel: lvl, tagsConfig: tagsConfig, } + + cfg.setMinLogLevel() + + return cfg +} + +func (l *logLevelConfig) setMinLogLevel() { + l.tagsConfig.tagLogs.Range(func(_, value any) bool { + v := value.(zapcore.Level) + if v < l.logLevel.Level() { + l.logLevel.SetLevel(v) + } + return true + }) } func (l *logLevelConfig) update(cfg *viper.Viper, log *zap.Logger) { @@ -159,17 +175,23 @@ func (l *logLevelConfig) update(cfg *viper.Viper, log *zap.Logger) { if err := l.tagsConfig.update(cfg, l.logLevel.Level()); err != nil { log.Warn(logs.TagsLogConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) } + + l.setMinLogLevel() } func (t *tagsConfig) LevelEnabled(tag string, tgtLevel zapcore.Level) bool { lvl, ok := t.tagLogs.Load(tag) if !ok { - return false + return t.defaultLvl.Enabled(tgtLevel) } return lvl.(zapcore.Level).Enabled(tgtLevel) } +func (t *tagsConfig) DefaultEnabled(lvl zapcore.Level) bool { + return t.defaultLvl.Enabled(lvl) +} + func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error { tags, err := fetchLogTagsConfig(cfg, ll) if err != nil { @@ -194,6 +216,7 @@ func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error { for k, v := range tags { t.tagLogs.Store(k, v) } + t.defaultLvl.SetLevel(ll) return nil } diff --git a/cmd/http-gw/logger.go b/cmd/http-gw/logger.go index 91105f7..195aa4e 100644 --- a/cmd/http-gw/logger.go +++ b/cmd/http-gw/logger.go @@ -40,7 +40,8 @@ type zapCoreTagFilterWrapper struct { } type TagFilterSettings interface { - LevelEnabled(tag string, lvl zapcore.Level) bool + LevelEnabled(tag string, tgtLevel zapcore.Level) bool + DefaultEnabled(lvl zapcore.Level) bool } func (c *zapCoreTagFilterWrapper) Enabled(level zapcore.Level) bool { @@ -63,24 +64,26 @@ func (c *zapCoreTagFilterWrapper) Check(entry zapcore.Entry, checked *zapcore.Ch } func (c *zapCoreTagFilterWrapper) Write(entry zapcore.Entry, fields []zapcore.Field) error { - if c.shouldSkip(entry, fields) || c.shouldSkip(entry, c.extra) { + if c.shouldSkip(entry, fields, c.extra) { return nil } return c.core.Write(entry, fields) } -func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field) bool { +func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field, extra []zap.Field) bool { for _, field := range fields { if field.Key == logs.TagFieldName && field.Type == zapcore.StringType { - if !c.settings.LevelEnabled(field.String, entry.Level) { - return true - } - break + return !c.settings.LevelEnabled(field.String, entry.Level) + } + } + for _, field := range extra { + if field.Key == logs.TagFieldName && field.Type == zapcore.StringType { + return !c.settings.LevelEnabled(field.String, entry.Level) } } - return false + return !c.settings.DefaultEnabled(entry.Level) } func (c *zapCoreTagFilterWrapper) Sync() error { diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 132c627..982b401 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -113,7 +113,7 @@ const ( cfgLoggerTags = "logger.tags" cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d." - cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "name" + cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "names" cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level" // Wallet. @@ -516,8 +516,8 @@ func fetchLogTagsConfig(v *viper.Viper, defaultLvl zapcore.Level) (map[string]za res := make(map[string]zapcore.Level) for i := 0; ; i++ { - name := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i)) - if name == "" { + tagNames := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i)) + if tagNames == "" { break } @@ -529,7 +529,12 @@ func fetchLogTagsConfig(v *viper.Viper, defaultLvl zapcore.Level) (map[string]za } } - res[name] = lvl + for _, tagName := range strings.Split(tagNames, ",") { + tagName = strings.TrimSpace(tagName) + if len(tagName) != 0 { + res[tagName] = lvl + } + } } if len(res) == 0 && !v.IsSet(cfgLoggerTags) { diff --git a/config/config.env b/config/config.env index 0ff2dec..72492d8 100644 --- a/config/config.env +++ b/config/config.env @@ -20,8 +20,9 @@ HTTP_GW_LOGGER_SAMPLING_ENABLED=false HTTP_GW_LOGGER_SAMPLING_INITIAL=100 HTTP_GW_LOGGER_SAMPLING_THEREAFTER=100 HTTP_GW_LOGGER_SAMPLING_INTERVAL=1s -HTTP_GW_LOGGER_TAGS_0_NAME=app -HTTP_GW_LOGGER_TAGS_1_NAME=datapath +HTTP_GW_LOGGER_TAGS_0_NAMES=app,datapath +HTTP_GW_LOGGER_TAGS_0_LEVEL=level +HTTP_GW_LOGGER_TAGS_1_NAME=external_storage_tree HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443 HTTP_GW_SERVER_0_TLS_ENABLED=false diff --git a/config/config.yaml b/config/config.yaml index 05bba2e..ccd025e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -30,8 +30,7 @@ logger: thereafter: 100 interval: 1s tags: - - name: app - - name: datapath + - names: app,datapath level: debug server: diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 628d3c7..1dec574 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -176,10 +176,9 @@ logger: thereafter: 100 interval: 1s tags: - - name: "app" + - names: "app,datapath" level: info - - name: "datapath" - - name: "external_storage_tree" + - names: "external_storage_tree" ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -199,14 +198,14 @@ parameter. Available tags: ```yaml tags: - - name: "app" + - names: "app,datapath" level: info ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-----------------------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------| -| `name` | `string` | yes | | Tag name. Possible values see below in `Tag values` section. | -| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------| +| `names` | `[]string` | yes | | Tag names separated by `,`. Possible values see below in `Tag values` section. | +| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | ### Tag values From 273459e0904b97929143a0237d59222419bc59ee Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 7 Apr 2025 16:50:48 +0300 Subject: [PATCH 77/86] [#225] Support wildcard in allowed origins and headers Signed-off-by: Marina Biryukova --- internal/handler/cors.go | 29 +- internal/handler/cors_test.go | 490 ++++++++++++++++++++++++++++++++++ 2 files changed, 510 insertions(+), 9 deletions(-) diff --git a/internal/handler/cors.go b/internal/handler/cors.go index d77ae02..bbfce1e 100644 --- a/internal/handler/cors.go +++ b/internal/handler/cors.go @@ -5,6 +5,8 @@ import ( "encoding/xml" "errors" "fmt" + "regexp" + "slices" "sort" "strconv" "strings" @@ -78,7 +80,7 @@ func (h *Handler) Preflight(req *fasthttp.RequestCtx) { for _, rule := range corsConfig.CORSRules { for _, o := range rule.AllowedOrigins { - if o == string(origin) || o == wildcard { + if o == string(origin) || o == wildcard || (strings.Contains(o, "*") && match(o, string(origin))) { for _, m := range rule.AllowedMethods { if m == string(method) { if !checkSubslice(rule.AllowedHeaders, headers) { @@ -117,6 +119,11 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) { return } + method := req.Request.Header.Peek(fasthttp.HeaderAccessControlRequestMethod) + if len(method) == 0 { + method = req.Method() + } + ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) cidParam, _ := req.UserValue("cid").(string) reqLog := h.reqLogger(ctx) @@ -141,9 +148,9 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) { for _, rule := range corsConfig.CORSRules { for _, o := range rule.AllowedOrigins { - if o == string(origin) { + if o == string(origin) || (strings.Contains(o, "*") && len(o) > 1 && match(o, string(origin))) { for _, m := range rule.AllowedMethods { - if m == string(req.Method()) { + if m == string(method) { req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) req.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") @@ -154,7 +161,7 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) { } if o == wildcard { for _, m := range rule.AllowedMethods { - if m == string(req.Method()) { + if m == string(method) { if withCredentials { req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") @@ -318,12 +325,9 @@ func setCORSHeadersFromRule(c *fasthttp.RequestCtx, cors *data.CORSRule) { } func checkSubslice(slice []string, subSlice []string) bool { - if sliceContains(slice, wildcard) { + if slices.Contains(slice, wildcard) { return true } - if len(subSlice) > len(slice) { - return false - } for _, r := range subSlice { if !sliceContains(slice, r) { return false @@ -334,9 +338,16 @@ func checkSubslice(slice []string, subSlice []string) bool { func sliceContains(slice []string, str string) bool { for _, s := range slice { - if s == str { + if s == str || (strings.Contains(s, "*") && match(s, str)) { return true } } return false } + +func match(tmpl, str string) bool { + regexpStr := "^" + regexp.QuoteMeta(tmpl) + "$" + regexpStr = regexpStr[:strings.Index(regexpStr, "*")-1] + "." + regexpStr[strings.Index(regexpStr, "*"):] + reg := regexp.MustCompile(regexpStr) + return reg.Match([]byte(str)) +} diff --git a/internal/handler/cors_test.go b/internal/handler/cors_test.go index 7cd7b0d..1ac07d7 100644 --- a/internal/handler/cors_test.go +++ b/internal/handler/cors_test.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/xml" "fmt" + "net/http" "testing" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" @@ -407,6 +408,12 @@ func TestCheckSubslice(t *testing.T) { actual: []string{"str1", "str5"}, expected: false, }, + { + name: "wildcard in allowed", + allowed: []string{"str*"}, + actual: []string{"str", "str5"}, + expected: true, + }, } { t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.expected, checkSubslice(tc.allowed, tc.actual)) @@ -414,6 +421,489 @@ func TestCheckSubslice(t *testing.T) { } } +func TestAllowedOriginWildcards(t *testing.T) { + hc := prepareHandlerContext(t) + bktName := "bucket-allowed-origin-wildcards" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + cfg := &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*suffix.example"}, + AllowedMethods: []string{"GET"}, + }, + { + AllowedOrigins: []string{"https://*example"}, + AllowedMethods: []string{"GET"}, + }, + { + AllowedOrigins: []string{"prefix.example*"}, + AllowedMethods: []string{"GET"}, + }, + }, + } + setCORSObject(t, hc, cnrID, cfg, 1) + + for _, tc := range []struct { + name string + handler func(*fasthttp.RequestCtx) + requestHeaders map[string]string + expectedHeaders map[string]string + expectedStatus int + }{ + { + name: "set cors headers, empty request cors headers", + handler: hc.Handler().SetCORSHeaders, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, invalid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://origin.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, first rule, no symbols in place of wildcard", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "suffix.example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "suffix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, first rule, valid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://suffix.example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://suffix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, first rule, invalid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://suffix-example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, second rule, no symbols in place of wildcard", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, second rule, valid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://www.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, second rule, invalid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, third rule, no symbols in place of wildcard", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "prefix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, third rule, valid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, third rule, invalid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "www.prefix.example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, third rule, invalid request method in header", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlRequestMethod: "PUT", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, third rule, valid request method in header", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, empty request cors headers", + handler: hc.Handler().Preflight, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusBadRequest, + }, + { + name: "preflight, invalid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://origin.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "preflight, first rule, no symbols in place of wildcard", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "suffix.example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "suffix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "prelight, first rule, valid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://suffix.example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://suffix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, first rule, invalid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://suffix-example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "preflight, second rule, no symbols in place of wildcard", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, second rule, valid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://www.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, second rule, invalid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "preflight, third rule, no symbols in place of wildcard", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "prefix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, third rule, valid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, third rule, invalid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "www.prefix.example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "preflight, third rule, invalid request method in header", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlRequestMethod: "PUT", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusForbidden, + }, + } { + t.Run(tc.name, func(t *testing.T) { + r := prepareCORSRequest(t, bktName, tc.requestHeaders) + tc.handler(r) + + expectedStatus := fasthttp.StatusOK + if tc.expectedStatus != 0 { + expectedStatus = tc.expectedStatus + } + require.Equal(t, expectedStatus, r.Response.StatusCode()) + for k, v := range tc.expectedHeaders { + require.Equal(t, v, string(r.Response.Header.Peek(k))) + } + }) + } +} + +func TestAllowedHeaderWildcards(t *testing.T) { + hc := prepareHandlerContext(t) + bktName := "bucket-allowed-header-wildcards" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + cfg := &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"https://www.example.com"}, + AllowedMethods: []string{"HEAD"}, + AllowedHeaders: []string{"*-suffix"}, + }, + { + AllowedOrigins: []string{"https://www.example.com"}, + AllowedMethods: []string{"HEAD"}, + AllowedHeaders: []string{"start-*-end"}, + }, + { + AllowedOrigins: []string{"https://www.example.com"}, + AllowedMethods: []string{"HEAD"}, + AllowedHeaders: []string{"X-Amz-*"}, + }, + }, + } + setCORSObject(t, hc, cnrID, cfg, 1) + + for _, tc := range []struct { + name string + requestHeaders map[string]string + expectedHeaders map[string]string + expectedStatus int + }{ + { + name: "first rule, valid headers", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "header-suffix, -suffix", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlAllowMethods: "HEAD", + fasthttp.HeaderAccessControlAllowHeaders: "header-suffix, -suffix", + }, + }, + { + name: "first rule, invalid headers", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "header-suffix-*", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "second rule, valid headers", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "start--end, start-header-end", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlAllowMethods: "HEAD", + fasthttp.HeaderAccessControlAllowHeaders: "start--end, start-header-end", + }, + }, + { + name: "second rule, invalid header ending", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "start-header-end-*", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "second rule, invalid header beginning", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "*-start-header-end", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "third rule, valid headers", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "X-Amz-Date, X-Amz-Content-Sha256", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlAllowMethods: "HEAD", + fasthttp.HeaderAccessControlAllowHeaders: "X-Amz-Date, X-Amz-Content-Sha256", + }, + }, + { + name: "third rule, invalid headers", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "Authorization", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + }, + expectedStatus: http.StatusForbidden, + }, + } { + t.Run(tc.name, func(t *testing.T) { + r := prepareCORSRequest(t, bktName, tc.requestHeaders) + hc.Handler().Preflight(r) + + expectedStatus := http.StatusOK + if tc.expectedStatus != 0 { + expectedStatus = tc.expectedStatus + } + require.Equal(t, expectedStatus, r.Response.StatusCode()) + for k, v := range tc.expectedHeaders { + require.Equal(t, v, string(r.Response.Header.Peek(k))) + } + }) + } +} + func setCORSObject(t *testing.T, hc *handlerContext, cnrID cid.ID, corsConfig *data.CORSConfiguration, epoch uint64) { payload, err := xml.Marshal(corsConfig) require.NoError(t, err) From 304dbdd4c8deeec5850bc07ae5cd28c2acd0b26e Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Wed, 16 Apr 2025 16:39:26 +0300 Subject: [PATCH 78/86] [#228] Update Go to 1.23 Signed-off-by: Nikita Zinkevich --- .docker/Dockerfile | 2 +- .forgejo/workflows/builds.yml | 2 +- .forgejo/workflows/tests.yml | 6 +++--- .forgejo/workflows/vulncheck.yml | 2 +- .golangci.yml | 3 --- CHANGELOG.md | 2 ++ Makefile | 8 ++++---- go.mod | 2 +- 8 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index f45c864..8d6f806 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22-alpine AS basebuilder +FROM golang:1.24-alpine AS basebuilder RUN apk add --update make bash ca-certificates FROM basebuilder AS builder diff --git a/.forgejo/workflows/builds.yml b/.forgejo/workflows/builds.yml index 7c2bb04..ebb6bcc 100644 --- a/.forgejo/workflows/builds.yml +++ b/.forgejo/workflows/builds.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.22', '1.23' ] + go_versions: [ '1.23', '1.24' ] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index d4182ed..8fb4c10 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.23' + go-version: '1.24' cache: true - name: Install linters @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.22', '1.23' ] + go_versions: [ '1.23', '1.24' ] fail-fast: false steps: - uses: actions/checkout@v3 @@ -53,7 +53,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.23' + go-version: '1.24' - name: Run integration tests run: |- diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 5fb9dc5..a58d2df 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -16,7 +16,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.23' check-latest: true - name: Install govulncheck diff --git a/.golangci.yml b/.golangci.yml index d9f93eb..2c754ac 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -22,9 +22,6 @@ linters-settings: # 'default' case is present, even if all enum members aren't listed in the # switch default-signifies-exhaustive: true - govet: - # report about shadowed variables - check-shadowing: false custom: truecloudlab-linters: path: bin/external_linters.so diff --git a/CHANGELOG.md b/CHANGELOG.md index 85798b8..4465d2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ This document outlines major changes between releases. ## [Unreleased] +- Update Go to 1.23 (#228) + ### Added - Add handling quota limit reached error (#187) - Add slash clipping for FileName attribute (#174) diff --git a/Makefile b/Makefile index 5b9e5bf..2218765 100755 --- a/Makefile +++ b/Makefile @@ -2,9 +2,9 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") -GO_VERSION ?= 1.22 -LINT_VERSION ?= 1.60.3 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.6 +GO_VERSION ?= 1.23 +LINT_VERSION ?= 1.64.8 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.10 BUILD ?= $(shell date -u --iso=seconds) HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs-http-gw @@ -150,7 +150,7 @@ dirty-image: @@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) @rm -rf $(TMP_DIR)/linters @rmdir $(TMP_DIR) 2>/dev/null || true - @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) + @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install -trimpath github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) # Run linters lint: diff --git a/go.mod b/go.mod index 31cf242..c065b57 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw -go 1.22 +go 1.23 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 From b9f1f455f8ce421ecb2f8ee5a24ffcf0d5026de2 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Wed, 16 Apr 2025 16:11:43 +0300 Subject: [PATCH 79/86] [#229] Add ngfuzz installation to makefile Signed-off-by: Marina Biryukova --- Makefile | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 2218765..11084f0 100755 --- a/Makefile +++ b/Makefile @@ -30,9 +30,10 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed "s/-/~/")-${OS_RELEASE} .PHONY: debpackage debclean -FUZZ_NGFUZZ_DIR ?= "" +FUZZING_DIR = $(shell pwd)/tests/fuzzing/files +NGFUZZ_REPO = https://gitflic.ru/project/yadro/ngfuzz.git FUZZ_TIMEOUT ?= 30 -FUZZ_FUNCTIONS ?= "all" +FUZZ_FUNCTIONS ?= "" FUZZ_AUX ?= "" # Make all binaries @@ -99,18 +100,22 @@ check-ngfuzz: exit 1; \ fi -.PHONY: install-fuzzing-deps -install-fuzzing-deps: check-clang check-ngfuzz +.PHONY: install-ngfuzz +install-ngfuzz: +ifeq (,$(wildcard $(FUZZING_DIR)/ngfuzz)) + @rm -rf $(FUZZING_DIR)/ngfuzz + @git clone $(NGFUZZ_REPO) $(FUZZING_DIR)/ngfuzz + @cd $(FUZZING_DIR)/ngfuzz && make +endif .PHONY: fuzz -fuzz: install-fuzzing-deps +fuzz: check-clang install-ngfuzz @START_PATH=$$(pwd); \ - ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \ - cd $(FUZZ_NGFUZZ_DIR) && \ - ./ngfuzz -clean && \ - ./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \ - ./ngfuzz -report - + ROOT_PATH=$$(realpath --relative-to=$(FUZZING_DIR)/ngfuzz $$START_PATH) ; \ + cd $(FUZZING_DIR)/ngfuzz && \ + ./bin/ngfuzz clean && \ + env CGO_ENABLED=1 ./bin/ngfuzz fuzz --funcs $(FUZZ_FUNCTIONS) --rootdir $$ROOT_PATH --timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \ + ./bin/ngfuzz coverage --rootdir $$ROOT_PATH # Reformat code fmt: From b7b08d9d828741ccad0169b32dae5c7230c27401 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Wed, 16 Apr 2025 17:53:49 +0300 Subject: [PATCH 80/86] [#230] Refactor logger tag configuration Signed-off-by: Pavel Pogodaev --- cmd/http-gw/app.go | 12 +++--------- cmd/http-gw/logger.go | 6 ++---- cmd/http-gw/settings.go | 1 - 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index ca7797f..ed16234 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -198,17 +198,11 @@ func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error { return err } - t.tagLogs.Range(func(key, value any) bool { + t.tagLogs.Range(func(key, _ any) bool { k := key.(string) - v := value.(zapcore.Level) - if lvl, ok := tags[k]; ok { - if lvl != v { - t.tagLogs.Store(key, lvl) - } - } else { + if _, ok := tags[k]; !ok { t.tagLogs.Delete(key) - delete(tags, k) } return true }) @@ -695,7 +689,7 @@ func (a *app) configReload(ctx context.Context) { return } - a.settings.logLevelConfig.update(a.cfg.settings, a.log) + a.settings.logLevelConfig.update(a.cfg.config(), a.log) if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil { a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) diff --git a/cmd/http-gw/logger.go b/cmd/http-gw/logger.go index 195aa4e..196cff3 100644 --- a/cmd/http-gw/logger.go +++ b/cmd/http-gw/logger.go @@ -40,7 +40,7 @@ type zapCoreTagFilterWrapper struct { } type TagFilterSettings interface { - LevelEnabled(tag string, tgtLevel zapcore.Level) bool + LevelEnabled(tag string, lvl zapcore.Level) bool DefaultEnabled(lvl zapcore.Level) bool } @@ -130,14 +130,13 @@ func newLogEncoder() zapcore.Encoder { // // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. func newStdoutLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) *Logger { - stdout := zapcore.AddSync(os.Stderr) + stdout := zapcore.AddSync(os.Stdout) consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, lvl) consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, loggerSettings, tagSetting) return &Logger{ logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), - lvl: lvl, } } @@ -155,7 +154,6 @@ func newJournaldLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings Logge return &Logger{ logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), - lvl: lvl, } } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 982b401..814a14e 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -208,7 +208,6 @@ var defaultTags = []string{logs.TagApp, logs.TagDatapath, logs.TagExternalStorag type Logger struct { logger *zap.Logger - lvl zap.AtomicLevel } type appCfg struct { From ee628617a36f3a7512a29414dabf7af173a06e5d Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Fri, 18 Apr 2025 14:34:16 +0300 Subject: [PATCH 81/86] [#227] Don't use bearer token with CORS container Signed-off-by: Marina Biryukova --- internal/handler/cors.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/handler/cors.go b/internal/handler/cors.go index bbfce1e..7e8db93 100644 --- a/internal/handler/cors.go +++ b/internal/handler/cors.go @@ -197,9 +197,6 @@ func (h *Handler) getCORSConfig(ctx context.Context, log *zap.Logger, cidStr str addr.SetContainer(h.corsCnrID) addr.SetObject(objID) corsObj, err := h.frostfs.GetObject(ctx, PrmObjectGet{ - PrmAuth: PrmAuth{ - BearerToken: bearerToken(ctx), - }, Address: addr, }) if err != nil { @@ -223,11 +220,7 @@ func (h *Handler) getLastCORSObject(ctx context.Context, cnrID cid.ID) (oid.ID, filters.AddRootFilter() filters.AddFilter(object.AttributeFilePath, fmt.Sprintf(corsFilePathTemplate, cnrID), object.MatchStringEqual) - prmAuth := PrmAuth{ - BearerToken: bearerToken(ctx), - } res, err := h.frostfs.SearchObjects(ctx, PrmObjectSearch{ - PrmAuth: prmAuth, Container: h.corsCnrID, Filters: filters, }) @@ -246,7 +239,6 @@ func (h *Handler) getLastCORSObject(ctx context.Context, cnrID cid.ID) (oid.ID, err = res.Iterate(func(id oid.ID) bool { addr.SetObject(id) obj, headErr = h.frostfs.HeadObject(ctx, PrmObjectHead{ - PrmAuth: prmAuth, Address: addr, }) if headErr != nil { From 9cb9d141463e6d3ad90826e6357cd6bc2d1b1655 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 22 Apr 2025 18:16:23 +0300 Subject: [PATCH 82/86] [#233] get/head: Middleware refactor Add: * search index.html * fallback by leading slash Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 31 +++--- cmd/http-gw/settings.go | 5 +- config/config.env | 2 + config/config.yaml | 2 + docs/gate-configuration.md | 69 ++++++------- internal/handler/browse.go | 29 ++---- internal/handler/download.go | 165 +++++++++++++++++++++++++++++-- internal/handler/handler.go | 63 ++++++------ internal/handler/handler_test.go | 17 ++-- internal/handler/head.go | 45 ++++++++- internal/logs/logs.go | 4 +- 11 files changed, 311 insertions(+), 121 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index ed16234..f603d3b 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -100,17 +100,18 @@ type ( workerPoolSize int logLevelConfig *logLevelConfig - mu sync.RWMutex - defaultTimestamp bool - archiveCompression bool - clientCut bool - returnIndexPage bool - indexPageTemplate string - bufferMaxSizeForPut uint64 - namespaceHeader string - defaultNamespaces []string - cors *data.CORSRule - enableFilepathFallback bool + mu sync.RWMutex + defaultTimestamp bool + archiveCompression bool + clientCut bool + returnIndexPage bool + indexPageTemplate string + bufferMaxSizeForPut uint64 + namespaceHeader string + defaultNamespaces []string + cors *data.CORSRule + enableFilepathFallback bool + enableFilepathSlashFallback bool } tagsConfig struct { @@ -296,6 +297,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { indexPage, indexEnabled := fetchIndexPageTemplate(v, l) cors := fetchCORSConfig(v) enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback) + enableFilepathSlashFallback := v.GetBool(cfgFeaturesEnableFilepathSlashFallback) s.mu.Lock() defer s.mu.Unlock() @@ -311,6 +313,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.indexPageTemplate = indexPage s.cors = cors s.enableFilepathFallback = enableFilepathFallback + s.enableFilepathSlashFallback = enableFilepathSlashFallback } func (s *loggerSettings) DroppedLogsInc() { @@ -421,6 +424,12 @@ func (s *appSettings) EnableFilepathFallback() bool { return s.enableFilepathFallback } +func (s *appSettings) EnableFilepathSlashFallback() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.enableFilepathSlashFallback +} + func (a *app) initResolver() { var err error a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 814a14e..07722de 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -180,8 +180,9 @@ const ( cfgMultinetSubnets = "multinet.subnets" // Feature. - cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" - cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" + cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" + cfgFeaturesEnableFilepathSlashFallback = "features.enable_filepath_slash_fallback" + cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" // Containers. cfgContainersCORS = "containers.cors" diff --git a/config/config.env b/config/config.env index 72492d8..a86f3e8 100644 --- a/config/config.env +++ b/config/config.env @@ -174,6 +174,8 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl # Enable using fallback path to search for a object by attribute HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false +# See description in docs/gate-configuration.md +HTTP_GW_FEATURES_ENABLE_FILEPATH_SLASH_FALLBACK=false # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true diff --git a/config/config.yaml b/config/config.yaml index ccd025e..bb01d47 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -192,6 +192,8 @@ multinet: features: # Enable using fallback path to search for a object by attribute enable_filepath_fallback: false + # See description in docs/gate-configuration.md + enable_filepath_slash_fallback: false # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service tree_pool_netmap_support: true diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 1dec574..3a058ae 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -8,7 +8,6 @@ There are some custom types used for brevity: * `duration` -- string consisting of a number and a suffix. Suffix examples include `s` (seconds), `m` (minutes), `ms` ( milliseconds). - # Reload on SIGHUP Some config values can be reloaded on SIGHUP signal. @@ -163,7 +162,6 @@ server: | `tls.cert_file` | `string` | yes | | Path to the TLS certificate. | | `tls.key_file` | `string` | yes | | Path to the key. | - # `logger` section ```yaml @@ -177,7 +175,7 @@ logger: interval: 1s tags: - names: "app,datapath" - level: info + level: info - names: "external_storage_tree" ``` @@ -235,7 +233,6 @@ web: | `stream_request_body` | `bool` | `true` | Enables request body streaming, and calls the handler sooner when given body is larger than the current limit. | | `max_request_body_size` | `int` | `4194304` | Maximum request body size. The server rejects requests with bodies exceeding this limit. | - # `upload-header` section ```yaml @@ -271,7 +268,6 @@ archive: |---------------|--------|---------------|---------------|------------------------------------------------------------------| | `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. | - # `pprof` section Contains configuration for the `pprof` profiler. @@ -320,14 +316,13 @@ tracing: ``` | Parameter | Type | SIGHUP reload | Default value | Description | -| ------------ | -------------------------------------- | ------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- | +|--------------|----------------------------------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| | `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | | `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | | `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | | `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | | `attributes` | [[]Attributes](#attributes-subsection) | yes | | An array of configurable attributes in key-value format. | - #### `attributes` subsection ```yaml @@ -338,12 +333,13 @@ tracing: value: value ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-----------------------|----------|---------------|---------------|----------------------------------------------------------| -| `key` | `string` | yes | | Attribute key. | -| `value` | `string` | yes | | Attribute value. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|---------------|------------------| +| `key` | `string` | yes | | Attribute key. | +| `value` | `string` | yes | | Attribute value. | # `runtime` section + Contains runtime parameters. ```yaml @@ -372,7 +368,6 @@ frostfs: | `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. | | `tree_pool_max_attempts` | `uint32` | no | `0` | Sets max attempt to make successful tree request. Value 0 means the number of attempts equals to number of nodes in pool. | - ### `cache` section ```yaml @@ -393,7 +388,6 @@ cache: | `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. | | `cors` | [Cache config](#cache-subsection) | `lifetime: 5m`
`size: 1000` | Cache which stores container CORS configurations. | - #### `cache` subsection ```yaml @@ -406,7 +400,6 @@ size: 1000 | `lifetime` | `duration` | depends on cache | Lifetime of entries in cache. | | `size` | `int` | depends on cache | LRU cache size. | - # `resolve_bucket` section Bucket name resolving parameters from and to container ID. @@ -417,10 +410,10 @@ resolve_bucket: default_namespaces: [ "", "root" ] ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|----------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------| -| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. | -| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|----------------------|------------|---------------|-----------------------|--------------------------------------------------| +| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. | +| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | # `index_page` section @@ -450,9 +443,9 @@ If values are not set, settings from CORS container will be used. ```yaml cors: allow_origin: "*" - allow_methods: ["GET", "HEAD"] - allow_headers: ["Authorization"] - expose_headers: ["*"] + allow_methods: [ "GET", "HEAD" ] + allow_headers: [ "Authorization" ] + expose_headers: [ "*" ] allow_credentials: false max_age: 600 ``` @@ -472,15 +465,15 @@ Configuration of multinet support. ```yaml multinet: - enabled: false - balancer: roundrobin - restrict: false - fallback_delay: 300ms - subnets: - - mask: 1.2.3.4/24 - source_ips: - - 1.2.3.4 - - 1.2.3.5 + enabled: false + balancer: roundrobin + restrict: false + fallback_delay: 300ms + subnets: + - mask: 1.2.3.4/24 + source_ips: + - 1.2.3.4 + - 1.2.3.5 ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -512,13 +505,15 @@ Contains parameters for enabling features. ```yaml features: enable_filepath_fallback: true + enable_filepath_slash_fallback: false tree_pool_netmap_support: true ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. | -| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-------------------------------------------|--------|---------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FileName` attribute if object with `FilePath` attribute wasn't found. | +| `features.enable_filepath_slash_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FilePath`/`FileName` with/without (depends on provided value in `FilePath`/`FileName`) if object with provided `FilePath`/`FileName` wasn't found. This fallback goes `before enable_filepath_fallback`. | +| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | # `containers` section @@ -529,6 +524,6 @@ containers: cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-------------|----------|---------------|---------------|-----------------------------------------| -| `cors` | `string` | no | | Container name for CORS configurations. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|---------------|-----------------------------------------| +| `cors` | `string` | no | | Container name for CORS configurations. | diff --git a/internal/handler/browse.go b/internal/handler/browse.go index ebe9004..e1fc59d 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -161,6 +160,7 @@ func urlencode(path string) string { type GetObjectsResponse struct { objects []ResponseObject hasErrors bool + isNative bool } func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { @@ -226,7 +226,8 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck log := h.reqLogger(ctx) dirs := make(map[string]struct{}) result := &GetObjectsResponse{ - objects: make([]ResponseObject, 0, 100), + objects: make([]ResponseObject, 0, 100), + isNative: true, } for objExt := range resp { if objExt.Error != nil { @@ -322,28 +323,16 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID, } type browseParams struct { - bucketInfo *data.BucketInfo - prefix string - isNative bool - listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error) + bucketInfo *data.BucketInfo + prefix string + objects *GetObjectsResponse } func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p browseParams) { const S3Protocol = "s3" const FrostfsProtocol = "frostfs" - ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( - zap.String("bucket", p.bucketInfo.Name), - zap.String("container", p.bucketInfo.CID.EncodeToString()), - zap.String("prefix", p.prefix), - )) - resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToListObjects, err) - return - } - - objects := resp.objects + objects := p.objects.objects sort.Slice(objects, func(i, j int) bool { if objects[i].IsDir == objects[j].IsDir { return objects[i].FileName < objects[j].FileName @@ -363,7 +352,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p } bucketName := p.bucketInfo.Name protocol := S3Protocol - if p.isNative { + if p.objects.isNative { bucketName = p.bucketInfo.CID.EncodeToString() protocol = FrostfsProtocol } @@ -372,7 +361,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p Prefix: p.prefix, Objects: objects, Protocol: protocol, - HasErrors: resp.hasErrors, + HasErrors: p.objects.hasErrors, }); err != nil { h.logAndSendError(ctx, req, logs.FailedToExecuteTemplate, err) return diff --git a/internal/handler/download.go b/internal/handler/download.go index 114bf34..301d10f 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "net/url" + "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" @@ -31,13 +32,18 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { cidParam := req.UserValue("cid").(string) oidParam := req.UserValue("oid").(string) - downloadParam := req.QueryArgs().GetBool("download") ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( zap.String("cid", cidParam), zap.String("oid", oidParam), )) + path, err := url.QueryUnescape(oidParam) + if err != nil { + h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err) + return + } + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) @@ -50,18 +56,159 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { return } - var objID oid.ID - if checkS3Err == nil && shouldDownload(oidParam, downloadParam) { - h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile) - } else if err = objID.DecodeString(oidParam); err == nil { - h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile) + prm := MiddlewareParam{ + Context: ctx, + Request: req, + BktInfo: bktInfo, + Path: path, + } + + indexPageEnabled := h.config.IndexPageEnabled() + + if checkS3Err == nil { + run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound), + Middleware{Func: h.byS3PathMiddleware(h.receiveFile, noopFormer), Enabled: true}, + Middleware{Func: h.byS3PathMiddleware(h.receiveFile, indexFormer), Enabled: indexPageEnabled}, + Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsS3), Enabled: indexPageEnabled}, + ) } else { - h.browseIndex(ctx, req, cidParam, oidParam, checkS3Err != nil) + slashFallbackEnabled := h.config.EnableFilepathSlashFallback() + fileNameFallbackEnabled := h.config.EnableFilepathFallback() + + run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound), + Middleware{Func: h.byAddressMiddleware(h.receiveFile), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, noopFormer), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled}, + Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsNative), Enabled: indexPageEnabled}, + ) } } -func shouldDownload(oidParam string, downloadParam bool) bool { - return !isDir(oidParam) || downloadParam +type MiddlewareFunc func(param MiddlewareParam) bool + +type MiddlewareParam struct { + Context context.Context + Request *fasthttp.RequestCtx + BktInfo *data.BucketInfo + Path string +} + +type Middleware struct { + Func MiddlewareFunc + Enabled bool +} + +func run(prm MiddlewareParam, defaultMiddleware MiddlewareFunc, middlewares ...Middleware) { + for _, m := range middlewares { + if m.Enabled && !m.Func(prm) { + return + } + } + + defaultMiddleware(prm) +} + +func indexFormer(path string) string { + indexPath := path + if indexPath != "" && !strings.HasSuffix(indexPath, "/") { + indexPath += "/" + } + + return indexPath + "index.html" +} + +func reverseLeadingSlash(path string) string { + if path == "" || path == "/" { + return path + } + + if path[0] == '/' { + return path[1:] + } + + return "/" + path +} + +func noopFormer(path string) string { + return path +} + +func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), pathFormer func(string) string) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byS3Path") + defer span.End() + + path := pathFormer(prm.Path) + + foundOID, err := h.tree.GetLatestVersion(ctx, &prm.BktInfo.CID, path) + if err == nil { + if foundOID.IsDeleteMarker { + h.logAndSendError(ctx, prm.Request, logs.IndexWasDeleted, ErrObjectNotFound) + return false + } + + addr := newAddress(prm.BktInfo.CID, foundOID.OID) + handler(ctx, prm.Request, addr) + return false + } + + if !errors.Is(err, layer.ErrNodeNotFound) { + h.logAndSendError(ctx, prm.Request, logs.FailedToGetLatestVersionOfIndexObject, err, zap.String("path", path)) + return false + } + + return true + } +} + +func (h *Handler) byAttributeSearchMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), attr string, pathFormer func(string) string) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAttributeSearch") + defer span.End() + + path := pathFormer(prm.Path) + + res, err := h.search(ctx, prm.BktInfo.CID, attr, path, object.MatchStringEqual) + if err != nil { + h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err) + return false + } + defer res.Close() + + buf := make([]oid.ID, 1) + n, err := res.Read(buf) + if err == nil && n > 0 { + addr := newAddress(prm.BktInfo.CID, buf[0]) + handler(ctx, prm.Request, addr) + return false + } + + if !errors.Is(err, io.EOF) { + h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err) + return false + } + + return true + } +} + +func (h *Handler) byAddressMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAddress") + defer span.End() + + var objID oid.ID + if objID.DecodeString(prm.Path) == nil { + handler(ctx, prm.Request, newAddress(prm.BktInfo.CID, objID)) + return false + } + + return true + } } // DownloadByAttribute handles attribute-based download requests. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index a982bc2..b0daf44 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -35,6 +35,7 @@ type Config interface { BufferMaxSizeForPut() uint64 NamespaceHeader() string EnableFilepathFallback() bool + EnableFilepathSlashFallback() bool FormContainerZone(string) string CORS() *data.CORSRule } @@ -216,11 +217,11 @@ func (h *Handler) byNativeAddress(ctx context.Context, req *fasthttp.RequestCtx, // byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that // resolves object address from S3-like path /. -func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { +func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, bktInfo *data.BucketInfo, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path") defer span.End() - foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) + foundOID, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, path) if err != nil { h.logAndSendError(ctx, req, logs.FailedToGetLatestVersionOfObject, err, zap.String("path", path)) return @@ -230,7 +231,7 @@ func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID return } - addr := newAddress(cnrID, foundOID.OID) + addr := newAddress(bktInfo.CID, foundOID.OID) handler(ctx, req, addr) } @@ -418,37 +419,31 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } -func (h *Handler) browseIndex(ctx context.Context, req *fasthttp.RequestCtx, cidParam, oidParam string, isNativeList bool) { - ctx, span := tracing.StartSpanFromContext(ctx, "handler.browseIndex") - defer span.End() +type ListFunc func(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) - if !h.config.IndexPageEnabled() { - req.SetStatusCode(fasthttp.StatusNotFound) - return +func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex") + defer span.End() + + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( + zap.String("bucket", prm.BktInfo.Name), + zap.String("container", prm.BktInfo.CID.EncodeToString()), + zap.String("prefix", prm.Path), + )) + + objects, err := fn(ctx, prm.BktInfo, prm.Path) + if err != nil { + h.logAndSendError(ctx, prm.Request, logs.FailedToListObjects, err) + return false + } + + h.browseObjects(ctx, prm.Request, browseParams{ + bucketInfo: prm.BktInfo, + prefix: prm.Path, + objects: objects, + }) + + return false } - - unescapedKey, err := url.QueryUnescape(oidParam) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToUnescapeOIDParam, err) - return - } - - bktInfo, err := h.getBucketInfo(ctx, cidParam) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) - return - } - - listFunc := h.getDirObjectsS3 - if isNativeList { - // tree probe failed, trying to use native - listFunc = h.getDirObjectsNative - } - - h.browseObjects(ctx, req, browseParams{ - bucketInfo: bktInfo, - prefix: unescapedKey, - listObjects: listFunc, - isNative: isNativeList, - }) } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 93cb1d9..fc75d69 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -62,8 +62,9 @@ func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*d } type configMock struct { - additionalSearch bool - cors *data.CORSRule + additionalFilenameSearch bool + additionalSlashSearch bool + cors *data.CORSRule } func (c *configMock) DefaultTimestamp() bool { @@ -99,7 +100,11 @@ func (c *configMock) NamespaceHeader() string { } func (c *configMock) EnableFilepathFallback() bool { - return c.additionalSearch + return c.additionalFilenameSearch +} + +func (c *configMock) EnableFilepathSlashFallback() bool { + return c.additionalSlashSearch } func (c *configMock) FormContainerZone(string) string { @@ -327,7 +332,7 @@ func TestBasic(t *testing.T) { func TestFindObjectByAttribute(t *testing.T) { hc := prepareHandlerContext(t) - hc.cfg.additionalSearch = true + hc.cfg.additionalFilenameSearch = true bktName := "bucket" cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) @@ -407,7 +412,7 @@ func TestFindObjectByAttribute(t *testing.T) { t.Run(tc.name, func(t *testing.T) { obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] obj.SetAttributes(tc.firstAttr, tc.secondAttr) - hc.cfg.additionalSearch = tc.additionalSearch + hc.cfg.additionalFilenameSearch = tc.additionalSearch objID, err := hc.Handler().findObjectByAttribute(ctx, cnrID, tc.reqAttrKey, tc.reqAttrValue) if tc.err != "" { @@ -476,7 +481,7 @@ func TestNeedSearchByFileName(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - hc.cfg.additionalSearch = tc.additionalSearch + hc.cfg.additionalFilenameSearch = tc.additionalSearch res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal) require.Equal(t, tc.expected, res) diff --git a/internal/handler/head.go b/internal/handler/head.go index 11d45fc..e130124 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -5,6 +5,7 @@ import ( "errors" "io" "net/http" + "net/url" "strconv" "time" @@ -128,6 +129,12 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { zap.String("oid", oidParam), )) + path, err := url.QueryUnescape(oidParam) + if err != nil { + h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err) + return + } + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) @@ -140,9 +147,38 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { return } + prm := MiddlewareParam{ + Context: ctx, + Request: req, + BktInfo: bktInfo, + Path: path, + } + + indexPageEnabled := h.config.IndexPageEnabled() + + if checkS3Err == nil { + run(prm, h.errorMiddleware(logs.ObjectNotFound, layer.ErrNodeNotFound), + Middleware{Func: h.byS3PathMiddleware(h.headObject, noopFormer), Enabled: true}, + Middleware{Func: h.byS3PathMiddleware(h.headObject, indexFormer), Enabled: indexPageEnabled}, + ) + } else { + slashFallbackEnabled := h.config.EnableFilepathSlashFallback() + fileNameFallbackEnabled := h.config.EnableFilepathFallback() + + run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound), + Middleware{Func: h.byAddressMiddleware(h.headObject), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, noopFormer), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled}, + ) + } + var objID oid.ID if checkS3Err == nil { - h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject) + h.byS3Path(ctx, req, bktInfo, oidParam, h.headObject) } else if err = objID.DecodeString(oidParam); err == nil { h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { @@ -157,3 +193,10 @@ func (h *Handler) HeadByAttribute(req *fasthttp.RequestCtx) { h.byAttribute(ctx, req, h.headObject) } + +func (h *Handler) errorMiddleware(msg string, err error) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + h.logAndSendError(prm.Context, prm.Request, msg, err) + return false + } +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 3e9b931..213e7c7 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -108,7 +108,9 @@ const ( FailedToGetBucketInfo = "could not get bucket info" FailedToSubmitTaskToPool = "failed to submit task to pool" ObjectWasDeleted = "object was deleted" + IndexWasDeleted = "index was deleted" FailedToGetLatestVersionOfObject = "failed to get latest version of object" + FailedToGetLatestVersionOfIndexObject = "failed to get latest version of index object" FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists" FailedToListObjects = "failed to list objects" FailedToParseTemplate = "failed to parse template" @@ -118,7 +120,7 @@ const ( FailedToGetObject = "failed to get object" FailedToGetObjectPayload = "failed to get object payload" FailedToFindObjectByAttribute = "failed to get find object by attribute" - FailedToUnescapeOIDParam = "failed to unescape oid param" + FailedToUnescapePath = "failed to unescape path" InvalidOIDParam = "invalid oid param" CouldNotGetCORSConfiguration = "could not get cors configuration" EmptyOriginRequestHeader = "empty Origin request header" From 0b9b23e67c2daf35bda7254610ce0d95d2233301 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 23 Apr 2025 09:18:21 +0300 Subject: [PATCH 83/86] [#233] Make search by attribute as it is Signed-off-by: Denis Kirillov --- internal/handler/handler.go | 42 ---------------- internal/handler/handler_test.go | 84 -------------------------------- internal/logs/logs.go | 7 ++- 3 files changed, 3 insertions(+), 130 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index b0daf44..59a19ed 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -253,8 +253,6 @@ func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, han return } - val = prepareAtribute(key, val) - ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val))) @@ -292,10 +290,6 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, cnrID cid.ID, attrK n, err := res.Read(buf) if n == 0 { switch { - case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): - h.reqLogger(ctx).Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) - attrVal = prepareAtribute(attrFileName, attrVal) - return h.findObjectByAttribute(ctx, cnrID, attrFileName, attrVal) case errors.Is(err, io.EOF): h.reqLogger(ctx).Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) @@ -308,42 +302,6 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, cnrID cid.ID, attrK return buf[0], nil } -func (h *Handler) needSearchByFileName(key, val string) bool { - if key != attrFilePath || !h.config.EnableFilepathFallback() { - return false - } - - return strings.HasPrefix(val, "/") && strings.Count(val, "/") == 1 || !strings.Contains(val, "/") -} - -func prepareAtribute(attrKey, attrVal string) string { - if attrKey == attrFileName { - return prepareFileName(attrVal) - } - - if attrKey == attrFilePath { - return prepareFilePath(attrVal) - } - - return attrVal -} - -func prepareFileName(fileName string) string { - if strings.HasPrefix(fileName, "/") { - return fileName[1:] - } - - return fileName -} - -func prepareFilePath(filePath string) string { - if !strings.HasPrefix(filePath, "/") { - return "/" + filePath - } - - return filePath -} - // resolveContainer decode container id, if it's not a valid container id // then trey to resolve name using provided resolver. func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index fc75d69..81d9784 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -427,90 +427,6 @@ func TestFindObjectByAttribute(t *testing.T) { } } -func TestNeedSearchByFileName(t *testing.T) { - hc := prepareHandlerContext(t) - - for _, tc := range []struct { - name string - attrKey string - attrVal string - additionalSearch bool - expected bool - }{ - { - name: "need search - not contains slash", - attrKey: attrFilePath, - attrVal: "cat.png", - additionalSearch: true, - expected: true, - }, - { - name: "need search - single lead slash", - attrKey: attrFilePath, - attrVal: "/cat.png", - additionalSearch: true, - expected: true, - }, - { - name: "don't need search - single slash but not lead", - attrKey: attrFilePath, - attrVal: "cats/cat.png", - additionalSearch: true, - expected: false, - }, - { - name: "don't need search - more one slash", - attrKey: attrFilePath, - attrVal: "/cats/cat.png", - additionalSearch: true, - expected: false, - }, - { - name: "don't need search - incorrect attribute key", - attrKey: attrFileName, - attrVal: "cat.png", - additionalSearch: true, - expected: false, - }, - { - name: "don't need search - additional search disabled", - attrKey: attrFilePath, - attrVal: "cat.png", - additionalSearch: false, - expected: false, - }, - } { - t.Run(tc.name, func(t *testing.T) { - hc.cfg.additionalFilenameSearch = tc.additionalSearch - - res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal) - require.Equal(t, tc.expected, res) - }) - } -} - -func TestPrepareFileName(t *testing.T) { - fileName := "/cat.jpg" - expected := "cat.jpg" - actual := prepareFileName(fileName) - require.Equal(t, expected, actual) - - fileName = "cat.jpg" - actual = prepareFileName(fileName) - require.Equal(t, expected, actual) -} - -func TestPrepareFilePath(t *testing.T) { - filePath := "cat.jpg" - expected := "/cat.jpg" - actual := prepareFilePath(filePath) - require.Equal(t, expected, actual) - - filePath = "/cat.jpg" - actual = prepareFilePath(filePath) - require.Equal(t, expected, actual) -} - func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 213e7c7..e7d118f 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -131,10 +131,9 @@ const ( // Log messages with the "external_storage" tag. const ( - ObjectNotFound = "object not found" - ReadObjectListFailed = "read object list failed" - ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" - ObjectUploaded = "object uploaded" + ObjectNotFound = "object not found" + ReadObjectListFailed = "read object list failed" + ObjectUploaded = "object uploaded" ) // Log messages with the "external_storage_tree" tag. From e579549b41fd1f35824968a331143a48f1204550 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 23 Apr 2025 10:52:19 +0300 Subject: [PATCH 84/86] [#233] Add fallback tests Signed-off-by: Denis Kirillov --- internal/handler/handler_test.go | 281 +++++++++++++++++++++---------- internal/handler/head.go | 9 - 2 files changed, 193 insertions(+), 97 deletions(-) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 81d9784..dbb037d 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -26,6 +26,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/panjf2000/ants/v2" @@ -64,6 +65,7 @@ func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*d type configMock struct { additionalFilenameSearch bool additionalSlashSearch bool + indexEnabled bool cors *data.CORSRule } @@ -76,7 +78,7 @@ func (c *configMock) ArchiveCompression() bool { } func (c *configMock) IndexPageEnabled() bool { - return false + return c.indexEnabled } func (c *configMock) IndexPageTemplate() string { @@ -259,6 +261,7 @@ func TestBasic(t *testing.T) { err = json.Unmarshal(r.Response.Body(), &putRes) require.NoError(t, err) + hc.cfg.additionalFilenameSearch = true obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] fileName := prepareObjectAttributes(object.AttributeFileName, objFileName) filePath := prepareObjectAttributes(object.AttributeFilePath, objFilePath) @@ -269,6 +272,14 @@ func TestBasic(t *testing.T) { r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID) hc.Handler().DownloadByAddressOrBucketName(r) require.Equal(t, content, string(r.Response.Body())) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFilePath) + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, content, string(r.Response.Body())) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFileName) + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, content, string(r.Response.Body())) }) t.Run("head", func(t *testing.T) { @@ -276,6 +287,16 @@ func TestBasic(t *testing.T) { hc.Handler().HeadByAddressOrBucketName(r) require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFilePath) + hc.Handler().HeadByAddressOrBucketName(r) + require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) + require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFileName) + hc.Handler().HeadByAddressOrBucketName(r) + require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) + require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) }) t.Run("get by attribute", func(t *testing.T) { @@ -285,11 +306,11 @@ func TestBasic(t *testing.T) { r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath) hc.Handler().DownloadByAttribute(r) - require.Equal(t, content, string(r.Response.Body())) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName) hc.Handler().DownloadByAttribute(r) - require.Equal(t, content, string(r.Response.Body())) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) }) t.Run("head by attribute", func(t *testing.T) { @@ -300,13 +321,11 @@ func TestBasic(t *testing.T) { r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath) hc.Handler().HeadByAttribute(r) - require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) - require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName) hc.Handler().HeadByAttribute(r) - require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) - require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) }) t.Run("zip", func(t *testing.T) { @@ -330,101 +349,187 @@ func TestBasic(t *testing.T) { }) } -func TestFindObjectByAttribute(t *testing.T) { +func prepareHandlerAndBucket(t *testing.T) (*handlerContext, cid.ID) { hc := prepareHandlerContext(t) - hc.cfg.additionalFilenameSearch = true bktName := "bucket" cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) require.NoError(t, err) hc.frostfs.SetContainer(cnrID, cnr) - ctx := context.Background() - ctx = middleware.SetNamespace(ctx, "") + return hc, cnrID +} - content := "hello" - r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content) - require.NoError(t, err) +func TestGetObjectWithFallback(t *testing.T) { + ctx := middleware.SetNamespace(context.Background(), "") - hc.Handler().Upload(r) - require.Equal(t, r.Response.StatusCode(), http.StatusOK) + t.Run("by oid", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) - var putRes putResponse - err = json.Unmarshal(r.Response.Body(), &putRes) - require.NoError(t, err) + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 - testAttrVal1 := "/folder/cat.jpg" - testAttrVal2 := "cat.jpg" - testAttrVal3 := "test-attr-val3" + r := prepareGetRequest(ctx, cnrID.EncodeToString(), obj1ID.String()) + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) - for _, tc := range []struct { - name string - firstAttr object.Attribute - secondAttr object.Attribute - reqAttrKey string - reqAttrValue string - err string - additionalSearch bool - }{ - { - name: "success search by FileName", - firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), - secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), - reqAttrKey: attrFileName, - reqAttrValue: testAttrVal2, - additionalSearch: false, - }, - { - name: "failed search by FileName", - firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), - secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), - reqAttrKey: attrFileName, - reqAttrValue: testAttrVal3, - err: "not found", - additionalSearch: false, - }, - { - name: "success search by FilePath (with additional search)", - firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), - secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), - reqAttrKey: attrFilePath, - reqAttrValue: testAttrVal2, - additionalSearch: true, - }, - { - name: "failed by FilePath (with additional search)", - firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), - secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), - reqAttrKey: attrFilePath, - reqAttrValue: testAttrVal3, - err: "not found", - additionalSearch: true, - }, - { - name: "success search by FilePath with leading slash (with additional search)", - firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), - secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), - reqAttrKey: attrFilePath, - reqAttrValue: "/cat.jpg", - additionalSearch: true, - }, - } { - t.Run(tc.name, func(t *testing.T) { - obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] - obj.SetAttributes(tc.firstAttr, tc.secondAttr) - hc.cfg.additionalFilenameSearch = tc.additionalSearch + t.Run("by filepath as it is", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) - objID, err := hc.Handler().findObjectByAttribute(ctx, cnrID, tc.reqAttrKey, tc.reqAttrValue) - if tc.err != "" { - require.Error(t, err) - require.Contains(t, err.Error(), tc.err) - return - } + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 - require.NoError(t, err) - require.Equal(t, putRes.ObjectID, objID.EncodeToString()) - }) - } + obj2ID := oidtest.ID() + obj2 := object.New() + obj2.SetID(obj2ID) + obj2.SetPayload([]byte("obj2")) + obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/filepath/obj2")) + hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj2") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj2.Payload()), string(r.Response.Body())) + }) + + t.Run("by filepath slash fallback", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.additionalSlashSearch = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) + + t.Run("by filename fallback", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.additionalFilenameSearch = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) + + t.Run("by filename and slash fallback", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.additionalFilenameSearch = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.additionalSlashSearch = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) + + t.Run("index fallback", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/index.html")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.indexEnabled = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) + + t.Run("index filename fallback", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/index.html")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.indexEnabled = true + hc.cfg.additionalFilenameSearch = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) } func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { diff --git a/internal/handler/head.go b/internal/handler/head.go index e130124..e6d9a30 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -175,15 +175,6 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled}, ) } - - var objID oid.ID - if checkS3Err == nil { - h.byS3Path(ctx, req, bktInfo, oidParam, h.headObject) - } else if err = objID.DecodeString(oidParam); err == nil { - h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) - } else { - h.logAndSendError(ctx, req, logs.InvalidOIDParam, err) - } } // HeadByAttribute handles attribute-based head requests. From dbb1bcad00cf6a444e8a951d61755fd3766c481e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 23 Apr 2025 13:02:17 +0300 Subject: [PATCH 85/86] [#233] Fix browsing Simplify tree listing (we need only nodes in exactly the same parent level) Signed-off-by: Denis Kirillov --- docs/gate-configuration.md | 10 +- internal/handler/browse.go | 29 +++- internal/handler/download.go | 12 +- internal/handler/handler.go | 36 +---- internal/handler/handler_test.go | 131 ++++++++++++---- internal/handler/head.go | 6 +- .../handler/tree_service_client_mock_test.go | 141 ++++++++++++++++++ internal/handler/utils.go | 6 +- internal/layer/tree_service.go | 24 --- internal/templates/index.gotmpl | 12 +- tree/tree.go | 46 ++---- 11 files changed, 302 insertions(+), 151 deletions(-) create mode 100644 internal/handler/tree_service_client_mock_test.go delete mode 100644 internal/layer/tree_service.go diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 3a058ae..08e2679 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -509,11 +509,11 @@ features: tree_pool_netmap_support: true ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-------------------------------------------|--------|---------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FileName` attribute if object with `FilePath` attribute wasn't found. | -| `features.enable_filepath_slash_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FilePath`/`FileName` with/without (depends on provided value in `FilePath`/`FileName`) if object with provided `FilePath`/`FileName` wasn't found. This fallback goes `before enable_filepath_fallback`. | -| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-------------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FileName` attribute if object with `FilePath` attribute wasn't found. | +| `features.enable_filepath_slash_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FilePath`/`FileName` with/without (depends on provided value in `FilePath`/`FileName`) leading slash if object with provided `FilePath`/`FileName` wasn't found. This fallback goes before `enable_filepath_fallback`. | +| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | # `containers` section diff --git a/internal/handler/browse.go b/internal/handler/browse.go index e1fc59d..d9e6625 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -130,11 +130,15 @@ func parentDir(prefix string) string { return prefix[index:] } -func trimPrefix(encPrefix string) string { +func getParent(encPrefix string) string { prefix, err := url.PathUnescape(encPrefix) if err != nil { return "" } + if prefix != "" && prefix[len(prefix)-1] == '/' { + prefix = prefix[:len(prefix)-1] + } + slashIndex := strings.LastIndex(prefix, "/") if slashIndex == -1 { return "" @@ -164,7 +168,11 @@ type GetObjectsResponse struct { } func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { - nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) + if prefix != "" && prefix[len(prefix)-1] == '/' { + prefix = prefix[:len(prefix)-1] + } + + nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) if err != nil { return nil, err } @@ -185,7 +193,7 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn if obj.IsDeleteMarker { continue } - obj.FilePath = prefix + obj.FileName + obj.FilePath = prefix + "/" + obj.FileName obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath) result.objects = append(result.objects, obj) } @@ -194,9 +202,9 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn } func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { - var basePath string - if ind := strings.LastIndex(prefix, "/"); ind != -1 { - basePath = prefix[:ind+1] + basePath := prefix + if basePath != "" && basePath[len(basePath)-1] != '/' { + basePath += "/" } filters := object.NewSearchFilters() @@ -342,7 +350,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p tmpl, err := template.New("index").Funcs(template.FuncMap{ "formatSize": formatSize, - "trimPrefix": trimPrefix, + "getParent": getParent, "urlencode": urlencode, "parentDir": parentDir, }).Parse(h.config.IndexPageTemplate()) @@ -356,9 +364,14 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p bucketName = p.bucketInfo.CID.EncodeToString() protocol = FrostfsProtocol } + prefix := p.prefix + if prefix != "" && prefix[len(prefix)-1] != '/' { + prefix += "/" + } + if err = tmpl.Execute(req, &BrowsePageData{ Container: bucketName, - Prefix: p.prefix, + Prefix: prefix, Objects: objects, Protocol: protocol, HasErrors: p.objects.hasErrors, diff --git a/internal/handler/download.go b/internal/handler/download.go index 301d10f..15fb886 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -14,8 +14,8 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -51,7 +51,7 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) - if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) { h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) return } @@ -88,6 +88,8 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { } } +type ObjectHandlerFunc func(context.Context, *fasthttp.RequestCtx, oid.Address) + type MiddlewareFunc func(param MiddlewareParam) bool type MiddlewareParam struct { @@ -156,7 +158,7 @@ func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.Req return false } - if !errors.Is(err, layer.ErrNodeNotFound) { + if !errors.Is(err, tree.ErrNodeNotFound) { h.logAndSendError(ctx, prm.Request, logs.FailedToGetLatestVersionOfIndexObject, err, zap.String("path", path)) return false } @@ -165,7 +167,7 @@ func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.Req } } -func (h *Handler) byAttributeSearchMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), attr string, pathFormer func(string) string) MiddlewareFunc { +func (h *Handler) byAttributeSearchMiddleware(handler ObjectHandlerFunc, attr string, pathFormer func(string) string) MiddlewareFunc { return func(prm MiddlewareParam) bool { ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAttributeSearch") defer span.End() @@ -196,7 +198,7 @@ func (h *Handler) byAttributeSearchMiddleware(handler func(context.Context, *fas } } -func (h *Handler) byAddressMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) MiddlewareFunc { +func (h *Handler) byAddressMiddleware(handler ObjectHandlerFunc) MiddlewareFunc { return func(prm MiddlewareParam) bool { ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAddress") defer span.End() diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 59a19ed..4d1dc31 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -11,8 +11,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" @@ -173,7 +173,7 @@ type Handler struct { ownerID *user.ID config Config containerResolver ContainerResolver - tree layer.TreeService + tree *tree.Tree cache *cache.BucketCache workerPool *ants.Pool corsCnrID cid.ID @@ -190,7 +190,7 @@ type AppParams struct { CORSCache *cache.CORSCache } -func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler { +func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, @@ -205,36 +205,6 @@ func New(params *AppParams, config Config, tree layer.TreeService, workerPool *a } } -// byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that -// prepares request and object address to it. -func (h *Handler) byNativeAddress(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, objID oid.ID, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { - ctx, span := tracing.StartSpanFromContext(ctx, "handler.byNativeAddress") - defer span.End() - - addr := newAddress(cnrID, objID) - handler(ctx, req, addr) -} - -// byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that -// resolves object address from S3-like path /. -func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, bktInfo *data.BucketInfo, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { - ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path") - defer span.End() - - foundOID, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, path) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToGetLatestVersionOfObject, err, zap.String("path", path)) - return - } - if foundOID.IsDeleteMarker { - h.logAndSendError(ctx, req, logs.ObjectWasDeleted, ErrObjectNotFound) - return - } - - addr := newAddress(bktInfo.CID, foundOID.OID) - handler(ctx, req, addr) -} - // byAttribute is a wrapper similar to byNativeAddress. func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { cidParam, _ := req.UserValue("cid").(string) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index dbb037d..622940e 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -14,9 +14,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -36,32 +37,6 @@ import ( "go.uber.org/zap/zaptest" ) -type treeServiceMock struct { - system map[string]map[string]*data.BaseNodeVersion -} - -func newTreeService() *treeServiceMock { - return &treeServiceMock{ - system: make(map[string]map[string]*data.BaseNodeVersion), - } -} - -func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error { - _, ok := t.system["bucket-settings"] - if !ok { - return layer.ErrNodeNotFound - } - return nil -} - -func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeInfo, string, error) { - return nil, "", nil -} - -func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) { - return nil, nil -} - type configMock struct { additionalFilenameSearch bool additionalSlashSearch bool @@ -82,7 +57,7 @@ func (c *configMock) IndexPageEnabled() bool { } func (c *configMock) IndexPageTemplate() string { - return "" + return templates.DefaultIndexTemplate } func (c *configMock) IndexPageNativeTemplate() string { @@ -124,7 +99,7 @@ type handlerContext struct { h *Handler frostfs *TestFrostFS - tree *treeServiceMock + tree *treeServiceClientMock cfg *configMock } @@ -174,14 +149,14 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { }), } - treeMock := newTreeService() + treeMock := newTreeServiceClientMock() cfgMock := &configMock{} workerPool, err := ants.NewPool(1) if err != nil { return nil, err } - handler := New(params, cfgMock, treeMock, workerPool) + handler := New(params, cfgMock, tree.NewTree(treeMock, logger), workerPool) return &handlerContext{ key: key, @@ -532,6 +507,100 @@ func TestGetObjectWithFallback(t *testing.T) { }) } +func TestIndex(t *testing.T) { + ctx := middleware.SetNamespace(context.Background(), "") + + t.Run("s3", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + hc.tree.containers[cnrID.String()] = containerInfo{ + trees: map[string]map[string]nodeResponse{ + "system": {"bucket-settings": nodeResponse{nodeID: 1}}, + "version": { + "": nodeResponse{}, //root + "prefix": nodeResponse{ + nodeID: 1, + meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}}, + "obj1": nodeResponse{ + parentID: 1, + nodeID: 2, + meta: []nodeMeta{ + {key: tree.FileNameKey, value: []byte("obj1")}, + {key: "OID", value: []byte(obj1ID.String())}, + }, + }, + }, + }, + } + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.indexEnabled = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, "bucket", "dummy") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy") + }) + + t.Run("native", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.indexEnabled = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy") + }) +} + func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) diff --git a/internal/handler/head.go b/internal/handler/head.go index e6d9a30..508dc37 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -9,8 +9,8 @@ import ( "strconv" "time" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -142,7 +142,7 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) - if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) { h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) return } @@ -157,7 +157,7 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { indexPageEnabled := h.config.IndexPageEnabled() if checkS3Err == nil { - run(prm, h.errorMiddleware(logs.ObjectNotFound, layer.ErrNodeNotFound), + run(prm, h.errorMiddleware(logs.ObjectNotFound, tree.ErrNodeNotFound), Middleware{Func: h.byS3PathMiddleware(h.headObject, noopFormer), Enabled: true}, Middleware{Func: h.byS3PathMiddleware(h.headObject, indexFormer), Enabled: indexPageEnabled}, ) diff --git a/internal/handler/tree_service_client_mock_test.go b/internal/handler/tree_service_client_mock_test.go new file mode 100644 index 0000000..f3af52a --- /dev/null +++ b/internal/handler/tree_service_client_mock_test.go @@ -0,0 +1,141 @@ +package handler + +import ( + "context" + "errors" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" +) + +type nodeMeta struct { + key string + value []byte +} + +func (m nodeMeta) GetKey() string { + return m.key +} + +func (m nodeMeta) GetValue() []byte { + return m.value +} + +type nodeResponse struct { + meta []nodeMeta + nodeID uint64 + parentID uint64 + timestamp uint64 +} + +func (n nodeResponse) GetNodeID() []uint64 { + return []uint64{n.nodeID} +} + +func (n nodeResponse) GetParentID() []uint64 { + return []uint64{n.parentID} +} + +func (n nodeResponse) GetTimestamp() []uint64 { + return []uint64{n.timestamp} +} + +func (n nodeResponse) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.meta)) + for i, value := range n.meta { + res[i] = value + } + return res +} + +type containerInfo struct { + trees map[string]map[string]nodeResponse +} + +type treeServiceClientMock struct { + containers map[string]containerInfo +} + +func newTreeServiceClientMock() *treeServiceClientMock { + return &treeServiceClientMock{ + containers: make(map[string]containerInfo), + } +} + +func (t *treeServiceClientMock) GetNodes(_ context.Context, p *tree.GetNodesParams) ([]tree.NodeResponse, error) { + cnr, ok := t.containers[p.CnrID.EncodeToString()] + if !ok { + return nil, tree.ErrNodeNotFound + } + + tr, ok := cnr.trees[p.TreeID] + if !ok { + return nil, tree.ErrNodeNotFound + } + + node, ok := tr[strings.Join(p.Path, "/")] + if !ok { + return nil, tree.ErrNodeNotFound + } + + return []tree.NodeResponse{node}, nil +} + +func (t *treeServiceClientMock) GetSubTree(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, _ bool) ([]tree.NodeResponse, error) { + cnr, ok := t.containers[bktInfo.CID.EncodeToString()] + if !ok { + return nil, tree.ErrNodeNotFound + } + + tr, ok := cnr.trees[treeID] + if !ok { + return nil, tree.ErrNodeNotFound + } + + if len(rootID) != 1 { + return nil, errors.New("invalid rootID") + } + + var root *nodeResponse + for _, v := range tr { + if v.nodeID == rootID[0] { + root = &v + break + } + } + + if root == nil { + return nil, tree.ErrNodeNotFound + } + + var res []nodeResponse + if depth == 0 { + for _, v := range tr { + res = append(res, v) + } + } else { + res = append(res, *root) + depthIndex := 0 + for i := uint32(0); i < depth-1; i++ { + childrenCount := 0 + for _, v := range tr { + for j := range res[depthIndex:] { + if v.parentID == res[j].nodeID { + res = append(res, v) + childrenCount++ + break + } + } + } + depthIndex = len(res) - childrenCount + } + } + + res2 := make([]tree.NodeResponse, len(res)) + for i := range res { + res2[i] = res[i] + } + + return res2, nil +} diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 8cb070d..c17b878 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -6,9 +6,9 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -93,7 +93,7 @@ func formErrorResponse(err error) (string, int) { switch { case errors.Is(err, ErrAccessDenied): return fmt.Sprintf("Storage Access Denied:\n%v", err), fasthttp.StatusForbidden - case errors.Is(err, layer.ErrNodeAccessDenied): + case errors.Is(err, tree.ErrNodeAccessDenied): return fmt.Sprintf("Tree Access Denied:\n%v", err), fasthttp.StatusForbidden case errors.Is(err, ErrQuotaLimitReached): return fmt.Sprintf("Quota Reached:\n%v", err), fasthttp.StatusConflict @@ -101,7 +101,7 @@ func formErrorResponse(err error) (string, int) { return fmt.Sprintf("Container Not Found:\n%v", err), fasthttp.StatusNotFound case errors.Is(err, ErrObjectNotFound): return fmt.Sprintf("Object Not Found:\n%v", err), fasthttp.StatusNotFound - case errors.Is(err, layer.ErrNodeNotFound): + case errors.Is(err, tree.ErrNodeNotFound): return fmt.Sprintf("Tree Node Not Found:\n%v", err), fasthttp.StatusNotFound case errors.Is(err, ErrGatewayTimeout): return fmt.Sprintf("Gateway Timeout:\n%v", err), fasthttp.StatusGatewayTimeout diff --git a/internal/layer/tree_service.go b/internal/layer/tree_service.go deleted file mode 100644 index ff80543..0000000 --- a/internal/layer/tree_service.go +++ /dev/null @@ -1,24 +0,0 @@ -package layer - -import ( - "context" - "errors" - - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" -) - -// TreeService provide interface to interact with tree service using s3 data models. -type TreeService interface { - GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) - GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) - CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error -} - -var ( - // ErrNodeNotFound is returned from Tree service in case of not found error. - ErrNodeNotFound = errors.New("not found") - - // ErrNodeAccessDenied is returned from Tree service in case of access denied error. - ErrNodeAccessDenied = errors.New("access denied") -) diff --git a/internal/templates/index.gotmpl b/internal/templates/index.gotmpl index b14cc06..4c03404 100644 --- a/internal/templates/index.gotmpl +++ b/internal/templates/index.gotmpl @@ -1,11 +1,9 @@ {{$container := .Container}} -{{ $prefix := trimPrefix .Prefix }} - Index of {{.Protocol}}://{{$container}} - /{{if $prefix}}/{{$prefix}}/{{end}} + Index of {{.Protocol}}://{{$container}}/{{.Prefix}} -

Index of {{.Protocol}}://{{$container}}/{{if $prefix}}{{$prefix}}/{{end}}

+

Index of {{.Protocol}}://{{$container}}/{{.Prefix}}

{{ if .HasErrors }}
Errors occurred while processing the request. Perhaps some objects are missing @@ -57,11 +55,11 @@
- ⮐.. + ⮐..