Compare commits

..

2 commits

Author SHA1 Message Date
52481375b0 [#240] Fix native index page
All checks were successful
/ DCO (pull_request) Successful in 27s
/ Vulncheck (pull_request) Successful in 55s
/ Builds (pull_request) Successful in 1m36s
/ OCI image (pull_request) Successful in 2m22s
/ Lint (pull_request) Successful in 2m12s
/ Tests (pull_request) Successful in 57s
/ Integration tests (pull_request) Successful in 6m59s
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-05-06 12:20:49 +03:00
39cf4e508e [#240] Fix s3 index page
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-05-06 09:25:38 +03:00
10 changed files with 179 additions and 106 deletions

View file

@ -1,10 +1,7 @@
FROM golang:1.24 AS builder
RUN apt-get update && \
apt-get install --no-install-recommends -y \
bash \
ca-certificates \
&& \
rm -rf /var/lib/apt/lists/*
FROM golang:1.24-alpine AS basebuilder
RUN apk add --update make bash ca-certificates
FROM basebuilder AS builder
ENV GOGC=off
ENV CGO_ENABLED=0
ARG BUILD=now
@ -16,7 +13,7 @@ COPY . /src
RUN make
# Executable image
FROM debian:stable-slim
FROM scratch
WORKDIR /

View file

@ -1,10 +1,5 @@
FROM debian:stable-slim
RUN apt-get update && \
apt-get install --no-install-recommends -y \
bash \
ca-certificates \
&& \
rm -rf /var/lib/apt/lists/*
FROM alpine
RUN apk add --update --no-cache bash ca-certificates
WORKDIR /

View file

@ -31,10 +31,6 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
.PHONY: debpackage debclean
FUZZING_DIR = $(shell pwd)/tests/fuzzing/files
FUZZING_TEMP_DIRS = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/tempfuzz ")
FUZZING_COVER_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/cover.out ")
FUZZING_FUNC_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/func.txt ")
FUZZING_INDEX_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/index.html ")
NGFUZZ_REPO = https://gitflic.ru/project/yadro/ngfuzz.git
FUZZ_TIMEOUT ?= 30
FUZZ_FUNCTIONS ?= ""
@ -193,9 +189,6 @@ version:
clean:
rm -rf vendor
rm -rf $(BINDIR)
rm -rf $(FUZZING_DIR) $(FUZZING_TEMP_DIRS)
rm -f $(FUZZING_COVER_FILES) $(FUZZING_FUNC_FILES) $(FUZZING_INDEX_FILES)
git checkout -- go.mod go.sum
# Package for Debian
debpackage:

View file

@ -76,13 +76,13 @@ func newListObjectsResponseNative(attrs map[string]string) ResponseObject {
}
}
func getNextDir(filepath, prefix string) string {
func getNextDir(filepath, prefix string) *string {
restPath := strings.Replace(filepath, prefix, "", 1)
index := strings.Index(restPath, "/")
if index == -1 {
return ""
return nil
}
return restPath[:index]
return ptr(restPath[:index])
}
func lastPathElement(path string) string {
@ -143,15 +143,18 @@ func getParent(encPrefix string) string {
if slashIndex == -1 {
return ""
}
return prefix[:slashIndex]
return prefix[:slashIndex+1]
}
func urlencode(path string) string {
var res strings.Builder
prefixParts := strings.Split(path, "/")
for _, prefixPart := range prefixParts {
prefixPart = "/" + url.PathEscape(prefixPart)
for i, prefixPart := range prefixParts {
prefixPart = url.PathEscape(prefixPart)
if i != 0 {
prefixPart = "/" + prefixPart
}
if prefixPart == "/." || prefixPart == "/.." {
prefixPart = url.PathEscape(prefixPart)
}
@ -168,11 +171,16 @@ type GetObjectsResponse struct {
}
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
if prefix != "" && prefix[len(prefix)-1] == '/' {
prefix = prefix[:len(prefix)-1]
var treePrefix *string
if prefix != "" {
if prefix[len(prefix)-1] == '/' {
treePrefix = ptr(prefix[:len(prefix)-1])
} else {
treePrefix = &prefix
}
}
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, treePrefix, true)
if err != nil {
return nil, err
}
@ -193,14 +201,18 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
if obj.IsDeleteMarker {
continue
}
obj.FilePath = prefix + "/" + obj.FileName
obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath)
obj.FilePath = prefix + obj.FileName
obj.GetURL = "/get/" + bucketInfo.Name + "/" + urlencode(obj.FilePath)
result.objects = append(result.objects, obj)
}
return result, nil
}
func ptr(s string) *string {
return &s
}
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
basePath := prefix
if basePath != "" && basePath[len(basePath)-1] != '/' {
@ -247,7 +259,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck
if _, ok := dirs[objExt.Object.FileName]; ok {
continue
}
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + urlencode(objExt.Object.FilePath)
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + urlencode(objExt.Object.FilePath)
dirs[objExt.Object.FileName] = struct{}{}
} else {
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + objExt.Object.OID
@ -319,13 +331,13 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID,
}
dirname := getNextDir(attrs[object.AttributeFilePath], basePath)
if dirname == "" {
if dirname == nil {
return newListObjectsResponseNative(attrs), nil
}
return ResponseObject{
FileName: dirname,
FilePath: basePath + dirname,
FileName: *dirname,
FilePath: basePath + *dirname,
IsDir: true,
}, nil
}

View file

@ -325,11 +325,12 @@ func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex")
defer span.End()
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
h.reqLogger(ctx).Info(logs.BrowseIndex,
zap.String("bucket", prm.BktInfo.Name),
zap.String("container", prm.BktInfo.CID.EncodeToString()),
zap.String("prefix", prm.Path),
))
logs.TagField(logs.TagDatapath),
)
objects, err := fn(ctx, prm.BktInfo, prm.Path)
if err != nil {

View file

@ -24,6 +24,11 @@ import (
"go.uber.org/zap"
)
const (
fuzzSuccessExitCode = 0
fuzzFailExitCode = -1
)
func prepareStrings(tp *go_fuzz_utils.TypeProvider, count int) ([]string, error) {
array := make([]string, count)
var err error
@ -217,18 +222,23 @@ func InitFuzzUpload() {
}
func DoFuzzUpload(input []byte) {
func DoFuzzUpload(input []byte) int {
// FUZZER INIT
if len(input) < 100 {
return
return fuzzFailExitCode
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return
return fuzzFailExitCode
}
_, _, _, _, _, _, _, _ = upload(tp)
_, _, _, _, _, _, _, err = upload(tp)
if err != nil {
return fuzzFailExitCode
}
return fuzzSuccessExitCode
}
func FuzzUpload(f *testing.F) {
@ -297,28 +307,30 @@ func InitFuzzGet() {
}
func DoFuzzGet(input []byte) {
func DoFuzzGet(input []byte) int {
// FUZZER INIT
if len(input) < 100 {
return
return fuzzFailExitCode
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return
return fuzzFailExitCode
}
ctx, hc, cnrID, resp, filename, _, _, err := upload(tp)
if err != nil {
return
return fuzzFailExitCode
}
r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename)
if err != nil {
return
return fuzzFailExitCode
}
hc.Handler().DownloadByAddressOrBucketName(r)
return fuzzSuccessExitCode
}
func FuzzGet(f *testing.F) {
@ -331,28 +343,30 @@ func InitFuzzHead() {
}
func DoFuzzHead(input []byte) {
func DoFuzzHead(input []byte) int {
// FUZZER INIT
if len(input) < 100 {
return
return fuzzFailExitCode
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return
return fuzzFailExitCode
}
ctx, hc, cnrID, resp, filename, _, _, err := upload(tp)
if err != nil {
return
return fuzzFailExitCode
}
r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename)
if err != nil {
return
return fuzzFailExitCode
}
hc.Handler().HeadByAddressOrBucketName(r)
return fuzzSuccessExitCode
}
func FuzzHead(f *testing.F) {
@ -365,36 +379,36 @@ func InitFuzzDownloadByAttribute() {
}
func DoFuzzDownloadByAttribute(input []byte) {
func DoFuzzDownloadByAttribute(input []byte) int {
// FUZZER INIT
if len(input) < 100 {
return
return fuzzFailExitCode
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return
return fuzzFailExitCode
}
ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp)
if err != nil {
return
return fuzzFailExitCode
}
cid := cnrID.EncodeToString()
cid, err = maybeFillRandom(tp, cid)
if err != nil {
return
return fuzzFailExitCode
}
attrKey, err = maybeFillRandom(tp, attrKey)
if err != nil {
return
return fuzzFailExitCode
}
attrVal, err = maybeFillRandom(tp, attrVal)
if err != nil {
return
return fuzzFailExitCode
}
r := new(fasthttp.RequestCtx)
@ -404,6 +418,8 @@ func DoFuzzDownloadByAttribute(input []byte) {
r.SetUserValue("attr_val", attrVal)
hc.Handler().DownloadByAttribute(r)
return fuzzSuccessExitCode
}
func FuzzDownloadByAttribute(f *testing.F) {
@ -416,36 +432,36 @@ func InitFuzzHeadByAttribute() {
}
func DoFuzzHeadByAttribute(input []byte) {
func DoFuzzHeadByAttribute(input []byte) int {
// FUZZER INIT
if len(input) < 100 {
return
return fuzzFailExitCode
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return
return fuzzFailExitCode
}
ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp)
if err != nil {
return
return fuzzFailExitCode
}
cid := cnrID.EncodeToString()
cid, err = maybeFillRandom(tp, cid)
if err != nil {
return
return fuzzFailExitCode
}
attrKey, err = maybeFillRandom(tp, attrKey)
if err != nil {
return
return fuzzFailExitCode
}
attrVal, err = maybeFillRandom(tp, attrVal)
if err != nil {
return
return fuzzFailExitCode
}
r := new(fasthttp.RequestCtx)
@ -455,6 +471,8 @@ func DoFuzzHeadByAttribute(input []byte) {
r.SetUserValue("attr_val", attrVal)
hc.Handler().HeadByAttribute(r)
return fuzzSuccessExitCode
}
func FuzzHeadByAttribute(f *testing.F) {
@ -467,32 +485,32 @@ func InitFuzzDownloadZipped() {
}
func DoFuzzDownloadZipped(input []byte) {
func DoFuzzDownloadZipped(input []byte) int {
// FUZZER INIT
if len(input) < 100 {
return
return fuzzFailExitCode
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return
return fuzzFailExitCode
}
ctx, hc, cnrID, _, _, _, _, err := upload(tp)
if err != nil {
return
return fuzzFailExitCode
}
cid := cnrID.EncodeToString()
cid, err = maybeFillRandom(tp, cid)
if err != nil {
return
return fuzzFailExitCode
}
prefix := ""
prefix, err = maybeFillRandom(tp, prefix)
if err != nil {
return
return fuzzFailExitCode
}
r := new(fasthttp.RequestCtx)
@ -501,6 +519,8 @@ func DoFuzzDownloadZipped(input []byte) {
r.SetUserValue("prefix", prefix)
hc.Handler().DownloadZip(r)
return fuzzSuccessExitCode
}
func FuzzDownloadZipped(f *testing.F) {
@ -513,21 +533,21 @@ func InitFuzzStoreBearerTokenAppCtx() {
}
func DoFuzzStoreBearerTokenAppCtx(input []byte) {
func DoFuzzStoreBearerTokenAppCtx(input []byte) int {
// FUZZER INIT
if len(input) < 100 {
return
return fuzzFailExitCode
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return
return fuzzFailExitCode
}
prefix := ""
prefix, err = maybeFillRandom(tp, prefix)
if err != nil {
return
return fuzzFailExitCode
}
ctx := context.Background()
@ -550,6 +570,8 @@ func DoFuzzStoreBearerTokenAppCtx(input []byte) {
}
tokens.StoreBearerTokenAppCtx(ctx, r)
return fuzzSuccessExitCode
}
func FuzzStoreBearerTokenAppCtx(f *testing.F) {

View file

@ -520,15 +520,23 @@ func TestIndex(t *testing.T) {
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
obj2ID := oidtest.ID()
obj2 := object.New()
obj2.SetID(obj2ID)
obj2.SetPayload([]byte("obj2"))
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/.."))
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
hc.tree.containers[cnrID.String()] = containerInfo{
trees: map[string]map[string]nodeResponse{
"system": {"bucket-settings": nodeResponse{nodeID: 1}},
"version": {
"": nodeResponse{}, //root
"<root>": nodeResponse{}, //root
"prefix": nodeResponse{
nodeID: 1,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}},
"obj1": nodeResponse{
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}},
},
"prefix/obj1": nodeResponse{
parentID: 1,
nodeID: 2,
meta: []nodeMeta{
@ -536,6 +544,23 @@ func TestIndex(t *testing.T) {
{key: "OID", value: []byte(obj1ID.String())},
},
},
"": nodeResponse{
nodeID: 3,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("")}},
},
"/dir": nodeResponse{
parentID: 3,
nodeID: 4,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("dir")}},
},
"/dir/..": nodeResponse{
parentID: 4,
nodeID: 5,
meta: []nodeMeta{
{key: tree.FileNameKey, value: []byte("..")},
{key: "OID", value: []byte(obj2ID.String())},
},
},
},
},
}
@ -563,6 +588,21 @@ func TestIndex(t *testing.T) {
r = prepareGetRequest(ctx, "bucket", "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy")
r = prepareGetRequest(ctx, "bucket", "")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//">/</a>`)
r = prepareGetRequest(ctx, "bucket", "/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//dir/">dir/</a>`)
r = prepareGetRequest(ctx, "bucket", "/dir/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//dir%2F..">..</a>`)
})
t.Run("native", func(t *testing.T) {
@ -575,6 +615,13 @@ func TestIndex(t *testing.T) {
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
obj2ID := oidtest.ID()
obj2 := object.New()
obj2.SetID(obj2ID)
obj2.SetPayload([]byte("obj2"))
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/.."))
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
@ -598,6 +645,21 @@ func TestIndex(t *testing.T) {
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy")
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//">/</a>`)
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//dir/">dir/</a>`)
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/dir/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/`+obj2ID.String()+`">..</a>`)
})
}

View file

@ -127,6 +127,7 @@ const (
EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header"
CORSRuleWasNotMatched = "cors rule was not matched"
CouldntCacheCors = "couldn't cache cors"
BrowseIndex = "browse index"
)
// Log messages with the "external_storage" tag.

View file

@ -56,40 +56,24 @@
</thead>
<tbody>
{{ $parentPrefix := getParent .Prefix }}
{{if $parentPrefix }}
<tr>
<td>
⮐<a href="/get/{{$container}}{{ urlencode $parentPrefix }}/">..</a>
⮐<a href="/get/{{$container}}/{{ urlencode $parentPrefix }}">..</a>
</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
{{else}}
<tr>
<td>
⮐<a href="/get/{{$container}}/">..</a>
</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
{{end}}
{{range .Objects}}
<tr>
<td>
{{if .IsDir}}
🗀
<a href="{{.GetURL}}/">
{{.FileName}}/
</a>
<a href="{{.GetURL}}/">{{.FileName}}/</a>
{{else}}
🗎
<a href="{{ .GetURL }}">
{{.FileName}}
</a>
<a href="{{ .GetURL }}">{{.FileName}}</a>
{{end}}
</td>
<td>{{.OID}}</td>

View file

@ -323,17 +323,23 @@ func pathFromName(objectName string) []string {
return strings.Split(objectName, separator)
}
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) {
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix *string, latestOnly bool) ([]data.NodeInfo, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix")
defer span.End()
rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator))
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil
rootID := []uint64{0}
var err error
if prefix != nil {
rootID, err = c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(*prefix, separator))
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil
}
return nil, err
}
return nil, err
}
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {