diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 326efb2..8d6f806 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,10 +1,7 @@ -FROM golang:1.24 AS builder -RUN apt-get update && \ - apt-get install --no-install-recommends -y \ - bash \ - ca-certificates \ - && \ - rm -rf /var/lib/apt/lists/* +FROM golang:1.24-alpine AS basebuilder +RUN apk add --update make bash ca-certificates + +FROM basebuilder AS builder ENV GOGC=off ENV CGO_ENABLED=0 ARG BUILD=now @@ -16,7 +13,7 @@ COPY . /src RUN make # Executable image -FROM debian:stable-slim +FROM scratch WORKDIR / diff --git a/.docker/Dockerfile.dirty b/.docker/Dockerfile.dirty index 7acddba..f733447 100644 --- a/.docker/Dockerfile.dirty +++ b/.docker/Dockerfile.dirty @@ -1,10 +1,5 @@ -FROM debian:stable-slim -RUN apt-get update && \ - apt-get install --no-install-recommends -y \ - bash \ - ca-certificates \ - && \ - rm -rf /var/lib/apt/lists/* +FROM alpine +RUN apk add --update --no-cache bash ca-certificates WORKDIR / diff --git a/Makefile b/Makefile index 6dd2222..11084f0 100755 --- a/Makefile +++ b/Makefile @@ -31,10 +31,6 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ .PHONY: debpackage debclean FUZZING_DIR = $(shell pwd)/tests/fuzzing/files -FUZZING_TEMP_DIRS = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/tempfuzz ") -FUZZING_COVER_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/cover.out ") -FUZZING_FUNC_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/func.txt ") -FUZZING_INDEX_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/index.html ") NGFUZZ_REPO = https://gitflic.ru/project/yadro/ngfuzz.git FUZZ_TIMEOUT ?= 30 FUZZ_FUNCTIONS ?= "" @@ -192,10 +188,7 @@ version: # Clean up clean: rm -rf vendor - rm -rf $(BINDIR) - rm -rf $(FUZZING_DIR) $(FUZZING_TEMP_DIRS) - rm -f $(FUZZING_COVER_FILES) $(FUZZING_FUNC_FILES) $(FUZZING_INDEX_FILES) - git checkout -- go.mod go.sum + rm -rf $(BINDIR) # Package for Debian debpackage: diff --git a/internal/handler/browse.go b/internal/handler/browse.go index d9e6625..5296bab 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -76,13 +76,13 @@ func newListObjectsResponseNative(attrs map[string]string) ResponseObject { } } -func getNextDir(filepath, prefix string) string { +func getNextDir(filepath, prefix string) *string { restPath := strings.Replace(filepath, prefix, "", 1) index := strings.Index(restPath, "/") if index == -1 { - return "" + return nil } - return restPath[:index] + return ptr(restPath[:index]) } func lastPathElement(path string) string { @@ -143,15 +143,18 @@ func getParent(encPrefix string) string { if slashIndex == -1 { return "" } - return prefix[:slashIndex] + return prefix[:slashIndex+1] } func urlencode(path string) string { var res strings.Builder prefixParts := strings.Split(path, "/") - for _, prefixPart := range prefixParts { - prefixPart = "/" + url.PathEscape(prefixPart) + for i, prefixPart := range prefixParts { + prefixPart = url.PathEscape(prefixPart) + if i != 0 { + prefixPart = "/" + prefixPart + } if prefixPart == "/." || prefixPart == "/.." { prefixPart = url.PathEscape(prefixPart) } @@ -168,11 +171,16 @@ type GetObjectsResponse struct { } func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { - if prefix != "" && prefix[len(prefix)-1] == '/' { - prefix = prefix[:len(prefix)-1] + var treePrefix *string + if prefix != "" { + if prefix[len(prefix)-1] == '/' { + treePrefix = ptr(prefix[:len(prefix)-1]) + } else { + treePrefix = &prefix + } } - nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) + nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, treePrefix, true) if err != nil { return nil, err } @@ -193,14 +201,18 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn if obj.IsDeleteMarker { continue } - obj.FilePath = prefix + "/" + obj.FileName - obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath) + obj.FilePath = prefix + obj.FileName + obj.GetURL = "/get/" + bucketInfo.Name + "/" + urlencode(obj.FilePath) result.objects = append(result.objects, obj) } return result, nil } +func ptr(s string) *string { + return &s +} + func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { basePath := prefix if basePath != "" && basePath[len(basePath)-1] != '/' { @@ -247,7 +259,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck if _, ok := dirs[objExt.Object.FileName]; ok { continue } - objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + urlencode(objExt.Object.FilePath) + objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + urlencode(objExt.Object.FilePath) dirs[objExt.Object.FileName] = struct{}{} } else { objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + objExt.Object.OID @@ -319,13 +331,13 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID, } dirname := getNextDir(attrs[object.AttributeFilePath], basePath) - if dirname == "" { + if dirname == nil { return newListObjectsResponseNative(attrs), nil } return ResponseObject{ - FileName: dirname, - FilePath: basePath + dirname, + FileName: *dirname, + FilePath: basePath + *dirname, IsDir: true, }, nil } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2efd71d..75254de 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -325,11 +325,12 @@ func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc { ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex") defer span.End() - ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( + h.reqLogger(ctx).Info(logs.BrowseIndex, zap.String("bucket", prm.BktInfo.Name), zap.String("container", prm.BktInfo.CID.EncodeToString()), zap.String("prefix", prm.Path), - )) + logs.TagField(logs.TagDatapath), + ) objects, err := fn(ctx, prm.BktInfo, prm.Path) if err != nil { diff --git a/internal/handler/handler_fuzz_test.go b/internal/handler/handler_fuzz_test.go index 66d225b..ff38b11 100644 --- a/internal/handler/handler_fuzz_test.go +++ b/internal/handler/handler_fuzz_test.go @@ -24,6 +24,11 @@ import ( "go.uber.org/zap" ) +const ( + fuzzSuccessExitCode = 0 + fuzzFailExitCode = -1 +) + func prepareStrings(tp *go_fuzz_utils.TypeProvider, count int) ([]string, error) { array := make([]string, count) var err error @@ -217,18 +222,23 @@ func InitFuzzUpload() { } -func DoFuzzUpload(input []byte) { +func DoFuzzUpload(input []byte) int { // FUZZER INIT if len(input) < 100 { - return + return fuzzFailExitCode } tp, err := go_fuzz_utils.NewTypeProvider(input) if err != nil { - return + return fuzzFailExitCode } - _, _, _, _, _, _, _, _ = upload(tp) + _, _, _, _, _, _, _, err = upload(tp) + if err != nil { + return fuzzFailExitCode + } + + return fuzzSuccessExitCode } func FuzzUpload(f *testing.F) { @@ -297,28 +307,30 @@ func InitFuzzGet() { } -func DoFuzzGet(input []byte) { +func DoFuzzGet(input []byte) int { // FUZZER INIT if len(input) < 100 { - return + return fuzzFailExitCode } tp, err := go_fuzz_utils.NewTypeProvider(input) if err != nil { - return + return fuzzFailExitCode } ctx, hc, cnrID, resp, filename, _, _, err := upload(tp) if err != nil { - return + return fuzzFailExitCode } r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename) if err != nil { - return + return fuzzFailExitCode } hc.Handler().DownloadByAddressOrBucketName(r) + + return fuzzSuccessExitCode } func FuzzGet(f *testing.F) { @@ -331,28 +343,30 @@ func InitFuzzHead() { } -func DoFuzzHead(input []byte) { +func DoFuzzHead(input []byte) int { // FUZZER INIT if len(input) < 100 { - return + return fuzzFailExitCode } tp, err := go_fuzz_utils.NewTypeProvider(input) if err != nil { - return + return fuzzFailExitCode } ctx, hc, cnrID, resp, filename, _, _, err := upload(tp) if err != nil { - return + return fuzzFailExitCode } r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename) if err != nil { - return + return fuzzFailExitCode } hc.Handler().HeadByAddressOrBucketName(r) + + return fuzzSuccessExitCode } func FuzzHead(f *testing.F) { @@ -365,36 +379,36 @@ func InitFuzzDownloadByAttribute() { } -func DoFuzzDownloadByAttribute(input []byte) { +func DoFuzzDownloadByAttribute(input []byte) int { // FUZZER INIT if len(input) < 100 { - return + return fuzzFailExitCode } tp, err := go_fuzz_utils.NewTypeProvider(input) if err != nil { - return + return fuzzFailExitCode } ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp) if err != nil { - return + return fuzzFailExitCode } cid := cnrID.EncodeToString() cid, err = maybeFillRandom(tp, cid) if err != nil { - return + return fuzzFailExitCode } attrKey, err = maybeFillRandom(tp, attrKey) if err != nil { - return + return fuzzFailExitCode } attrVal, err = maybeFillRandom(tp, attrVal) if err != nil { - return + return fuzzFailExitCode } r := new(fasthttp.RequestCtx) @@ -404,6 +418,8 @@ func DoFuzzDownloadByAttribute(input []byte) { r.SetUserValue("attr_val", attrVal) hc.Handler().DownloadByAttribute(r) + + return fuzzSuccessExitCode } func FuzzDownloadByAttribute(f *testing.F) { @@ -416,36 +432,36 @@ func InitFuzzHeadByAttribute() { } -func DoFuzzHeadByAttribute(input []byte) { +func DoFuzzHeadByAttribute(input []byte) int { // FUZZER INIT if len(input) < 100 { - return + return fuzzFailExitCode } tp, err := go_fuzz_utils.NewTypeProvider(input) if err != nil { - return + return fuzzFailExitCode } ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp) if err != nil { - return + return fuzzFailExitCode } cid := cnrID.EncodeToString() cid, err = maybeFillRandom(tp, cid) if err != nil { - return + return fuzzFailExitCode } attrKey, err = maybeFillRandom(tp, attrKey) if err != nil { - return + return fuzzFailExitCode } attrVal, err = maybeFillRandom(tp, attrVal) if err != nil { - return + return fuzzFailExitCode } r := new(fasthttp.RequestCtx) @@ -455,6 +471,8 @@ func DoFuzzHeadByAttribute(input []byte) { r.SetUserValue("attr_val", attrVal) hc.Handler().HeadByAttribute(r) + + return fuzzSuccessExitCode } func FuzzHeadByAttribute(f *testing.F) { @@ -467,32 +485,32 @@ func InitFuzzDownloadZipped() { } -func DoFuzzDownloadZipped(input []byte) { +func DoFuzzDownloadZipped(input []byte) int { // FUZZER INIT if len(input) < 100 { - return + return fuzzFailExitCode } tp, err := go_fuzz_utils.NewTypeProvider(input) if err != nil { - return + return fuzzFailExitCode } ctx, hc, cnrID, _, _, _, _, err := upload(tp) if err != nil { - return + return fuzzFailExitCode } cid := cnrID.EncodeToString() cid, err = maybeFillRandom(tp, cid) if err != nil { - return + return fuzzFailExitCode } prefix := "" prefix, err = maybeFillRandom(tp, prefix) if err != nil { - return + return fuzzFailExitCode } r := new(fasthttp.RequestCtx) @@ -501,6 +519,8 @@ func DoFuzzDownloadZipped(input []byte) { r.SetUserValue("prefix", prefix) hc.Handler().DownloadZip(r) + + return fuzzSuccessExitCode } func FuzzDownloadZipped(f *testing.F) { @@ -513,21 +533,21 @@ func InitFuzzStoreBearerTokenAppCtx() { } -func DoFuzzStoreBearerTokenAppCtx(input []byte) { +func DoFuzzStoreBearerTokenAppCtx(input []byte) int { // FUZZER INIT if len(input) < 100 { - return + return fuzzFailExitCode } tp, err := go_fuzz_utils.NewTypeProvider(input) if err != nil { - return + return fuzzFailExitCode } prefix := "" prefix, err = maybeFillRandom(tp, prefix) if err != nil { - return + return fuzzFailExitCode } ctx := context.Background() @@ -550,6 +570,8 @@ func DoFuzzStoreBearerTokenAppCtx(input []byte) { } tokens.StoreBearerTokenAppCtx(ctx, r) + + return fuzzSuccessExitCode } func FuzzStoreBearerTokenAppCtx(f *testing.F) { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 6c715fe..eddb7c6 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -520,15 +520,23 @@ func TestIndex(t *testing.T) { obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1")) hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + obj2ID := oidtest.ID() + obj2 := object.New() + obj2.SetID(obj2ID) + obj2.SetPayload([]byte("obj2")) + obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/..")) + hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2 + hc.tree.containers[cnrID.String()] = containerInfo{ trees: map[string]map[string]nodeResponse{ "system": {"bucket-settings": nodeResponse{nodeID: 1}}, "version": { - "": nodeResponse{}, //root + "": nodeResponse{}, //root "prefix": nodeResponse{ nodeID: 1, - meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}}, - "obj1": nodeResponse{ + meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}, + }, + "prefix/obj1": nodeResponse{ parentID: 1, nodeID: 2, meta: []nodeMeta{ @@ -536,6 +544,23 @@ func TestIndex(t *testing.T) { {key: "OID", value: []byte(obj1ID.String())}, }, }, + "": nodeResponse{ + nodeID: 3, + meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("")}}, + }, + "/dir": nodeResponse{ + parentID: 3, + nodeID: 4, + meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("dir")}}, + }, + "/dir/..": nodeResponse{ + parentID: 4, + nodeID: 5, + meta: []nodeMeta{ + {key: tree.FileNameKey, value: []byte("..")}, + {key: "OID", value: []byte(obj2ID.String())}, + }, + }, }, }, } @@ -563,6 +588,21 @@ func TestIndex(t *testing.T) { r = prepareGetRequest(ctx, "bucket", "dummy") hc.Handler().DownloadByAddressOrBucketName(r) require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy") + + r = prepareGetRequest(ctx, "bucket", "") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), `..`) + require.Contains(t, string(r.Response.Body()), `/`) + + r = prepareGetRequest(ctx, "bucket", "/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), `..`) + require.Contains(t, string(r.Response.Body()), `dir/`) + + r = prepareGetRequest(ctx, "bucket", "/dir/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), `..`) + require.Contains(t, string(r.Response.Body()), `..`) }) t.Run("native", func(t *testing.T) { @@ -575,6 +615,13 @@ func TestIndex(t *testing.T) { obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1")) hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + obj2ID := oidtest.ID() + obj2 := object.New() + obj2.SetID(obj2ID) + obj2.SetPayload([]byte("obj2")) + obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/..")) + hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2 + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") hc.Handler().DownloadByAddressOrBucketName(r) require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) @@ -598,6 +645,21 @@ func TestIndex(t *testing.T) { r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy") hc.Handler().DownloadByAddressOrBucketName(r) require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy") + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), `..`) + require.Contains(t, string(r.Response.Body()), `/`) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), `..`) + require.Contains(t, string(r.Response.Body()), `dir/`) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/dir/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), `..`) + require.Contains(t, string(r.Response.Body()), `..`) }) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 86921dd..ee0806e 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -127,6 +127,7 @@ const ( EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header" CORSRuleWasNotMatched = "cors rule was not matched" CouldntCacheCors = "couldn't cache cors" + BrowseIndex = "browse index" ) // Log messages with the "external_storage" tag. diff --git a/internal/templates/index.gotmpl b/internal/templates/index.gotmpl index 4c03404..9ce0f5b 100644 --- a/internal/templates/index.gotmpl +++ b/internal/templates/index.gotmpl @@ -56,40 +56,24 @@ {{ $parentPrefix := getParent .Prefix }} - {{if $parentPrefix }} - ⮐.. + ⮐.. - {{else}} - - - ⮐.. - - - - - - - {{end}} {{range .Objects}} {{if .IsDir}} 🗀 - - {{.FileName}}/ - + {{.FileName}}/ {{else}} 🗎 - - {{.FileName}} - + {{.FileName}} {{end}} {{.OID}} diff --git a/tree/tree.go b/tree/tree.go index d99e24b..84529e8 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -323,17 +323,23 @@ func pathFromName(objectName string) []string { return strings.Split(objectName, separator) } -func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) { +func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix *string, latestOnly bool) ([]data.NodeInfo, error) { ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix") defer span.End() - rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator)) - if err != nil { - if errors.Is(err, ErrNodeNotFound) { - return nil, nil + rootID := []uint64{0} + var err error + + if prefix != nil { + rootID, err = c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(*prefix, separator)) + if err != nil { + if errors.Is(err, ErrNodeNotFound) { + return nil, nil + } + return nil, err } - return nil, err } + subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) if err != nil { if errors.Is(err, ErrNodeNotFound) {