Compare commits

..

3 commits

Author SHA1 Message Date
50021ce587 [#236] Switch base image for OCI container to Debian
Related: TrueCloudLab/frostfs-node#1724

Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>
2025-05-12 15:09:19 +00:00
5e6ee3a41c [#241] Add fuzzing files removing to make clean
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2025-05-07 10:35:37 +03:00
3aed60aa79 [#241] Remove exit codes from fuzzing tests
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2025-05-07 10:35:26 +03:00
10 changed files with 106 additions and 179 deletions

View file

@ -1,7 +1,10 @@
FROM golang:1.24-alpine AS basebuilder
RUN apk add --update make bash ca-certificates
FROM basebuilder AS builder
FROM golang:1.24 AS builder
RUN apt-get update && \
apt-get install --no-install-recommends -y \
bash \
ca-certificates \
&& \
rm -rf /var/lib/apt/lists/*
ENV GOGC=off
ENV CGO_ENABLED=0
ARG BUILD=now
@ -13,7 +16,7 @@ COPY . /src
RUN make
# Executable image
FROM scratch
FROM debian:stable-slim
WORKDIR /

View file

@ -1,5 +1,10 @@
FROM alpine
RUN apk add --update --no-cache bash ca-certificates
FROM debian:stable-slim
RUN apt-get update && \
apt-get install --no-install-recommends -y \
bash \
ca-certificates \
&& \
rm -rf /var/lib/apt/lists/*
WORKDIR /

View file

@ -31,6 +31,10 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
.PHONY: debpackage debclean
FUZZING_DIR = $(shell pwd)/tests/fuzzing/files
FUZZING_TEMP_DIRS = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/tempfuzz ")
FUZZING_COVER_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/cover.out ")
FUZZING_FUNC_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/func.txt ")
FUZZING_INDEX_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/index.html ")
NGFUZZ_REPO = https://gitflic.ru/project/yadro/ngfuzz.git
FUZZ_TIMEOUT ?= 30
FUZZ_FUNCTIONS ?= ""
@ -188,7 +192,10 @@ version:
# Clean up
clean:
rm -rf vendor
rm -rf $(BINDIR)
rm -rf $(BINDIR)
rm -rf $(FUZZING_DIR) $(FUZZING_TEMP_DIRS)
rm -f $(FUZZING_COVER_FILES) $(FUZZING_FUNC_FILES) $(FUZZING_INDEX_FILES)
git checkout -- go.mod go.sum
# Package for Debian
debpackage:

View file

@ -76,13 +76,13 @@ func newListObjectsResponseNative(attrs map[string]string) ResponseObject {
}
}
func getNextDir(filepath, prefix string) *string {
func getNextDir(filepath, prefix string) string {
restPath := strings.Replace(filepath, prefix, "", 1)
index := strings.Index(restPath, "/")
if index == -1 {
return nil
return ""
}
return ptr(restPath[:index])
return restPath[:index]
}
func lastPathElement(path string) string {
@ -143,18 +143,15 @@ func getParent(encPrefix string) string {
if slashIndex == -1 {
return ""
}
return prefix[:slashIndex+1]
return prefix[:slashIndex]
}
func urlencode(path string) string {
var res strings.Builder
prefixParts := strings.Split(path, "/")
for i, prefixPart := range prefixParts {
prefixPart = url.PathEscape(prefixPart)
if i != 0 {
prefixPart = "/" + prefixPart
}
for _, prefixPart := range prefixParts {
prefixPart = "/" + url.PathEscape(prefixPart)
if prefixPart == "/." || prefixPart == "/.." {
prefixPart = url.PathEscape(prefixPart)
}
@ -171,16 +168,11 @@ type GetObjectsResponse struct {
}
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
var treePrefix *string
if prefix != "" {
if prefix[len(prefix)-1] == '/' {
treePrefix = ptr(prefix[:len(prefix)-1])
} else {
treePrefix = &prefix
}
if prefix != "" && prefix[len(prefix)-1] == '/' {
prefix = prefix[:len(prefix)-1]
}
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, treePrefix, true)
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
if err != nil {
return nil, err
}
@ -201,18 +193,14 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
if obj.IsDeleteMarker {
continue
}
obj.FilePath = prefix + obj.FileName
obj.GetURL = "/get/" + bucketInfo.Name + "/" + urlencode(obj.FilePath)
obj.FilePath = prefix + "/" + obj.FileName
obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath)
result.objects = append(result.objects, obj)
}
return result, nil
}
func ptr(s string) *string {
return &s
}
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
basePath := prefix
if basePath != "" && basePath[len(basePath)-1] != '/' {
@ -259,7 +247,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck
if _, ok := dirs[objExt.Object.FileName]; ok {
continue
}
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + urlencode(objExt.Object.FilePath)
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + urlencode(objExt.Object.FilePath)
dirs[objExt.Object.FileName] = struct{}{}
} else {
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + objExt.Object.OID
@ -331,13 +319,13 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID,
}
dirname := getNextDir(attrs[object.AttributeFilePath], basePath)
if dirname == nil {
if dirname == "" {
return newListObjectsResponseNative(attrs), nil
}
return ResponseObject{
FileName: *dirname,
FilePath: basePath + *dirname,
FileName: dirname,
FilePath: basePath + dirname,
IsDir: true,
}, nil
}

View file

@ -325,12 +325,11 @@ func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex")
defer span.End()
h.reqLogger(ctx).Info(logs.BrowseIndex,
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
zap.String("bucket", prm.BktInfo.Name),
zap.String("container", prm.BktInfo.CID.EncodeToString()),
zap.String("prefix", prm.Path),
logs.TagField(logs.TagDatapath),
)
))
objects, err := fn(ctx, prm.BktInfo, prm.Path)
if err != nil {

View file

@ -24,11 +24,6 @@ import (
"go.uber.org/zap"
)
const (
fuzzSuccessExitCode = 0
fuzzFailExitCode = -1
)
func prepareStrings(tp *go_fuzz_utils.TypeProvider, count int) ([]string, error) {
array := make([]string, count)
var err error
@ -222,23 +217,18 @@ func InitFuzzUpload() {
}
func DoFuzzUpload(input []byte) int {
func DoFuzzUpload(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
_, _, _, _, _, _, _, err = upload(tp)
if err != nil {
return fuzzFailExitCode
}
return fuzzSuccessExitCode
_, _, _, _, _, _, _, _ = upload(tp)
}
func FuzzUpload(f *testing.F) {
@ -307,30 +297,28 @@ func InitFuzzGet() {
}
func DoFuzzGet(input []byte) int {
func DoFuzzGet(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
ctx, hc, cnrID, resp, filename, _, _, err := upload(tp)
if err != nil {
return fuzzFailExitCode
return
}
r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename)
if err != nil {
return fuzzFailExitCode
return
}
hc.Handler().DownloadByAddressOrBucketName(r)
return fuzzSuccessExitCode
}
func FuzzGet(f *testing.F) {
@ -343,30 +331,28 @@ func InitFuzzHead() {
}
func DoFuzzHead(input []byte) int {
func DoFuzzHead(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
ctx, hc, cnrID, resp, filename, _, _, err := upload(tp)
if err != nil {
return fuzzFailExitCode
return
}
r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename)
if err != nil {
return fuzzFailExitCode
return
}
hc.Handler().HeadByAddressOrBucketName(r)
return fuzzSuccessExitCode
}
func FuzzHead(f *testing.F) {
@ -379,36 +365,36 @@ func InitFuzzDownloadByAttribute() {
}
func DoFuzzDownloadByAttribute(input []byte) int {
func DoFuzzDownloadByAttribute(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp)
if err != nil {
return fuzzFailExitCode
return
}
cid := cnrID.EncodeToString()
cid, err = maybeFillRandom(tp, cid)
if err != nil {
return fuzzFailExitCode
return
}
attrKey, err = maybeFillRandom(tp, attrKey)
if err != nil {
return fuzzFailExitCode
return
}
attrVal, err = maybeFillRandom(tp, attrVal)
if err != nil {
return fuzzFailExitCode
return
}
r := new(fasthttp.RequestCtx)
@ -418,8 +404,6 @@ func DoFuzzDownloadByAttribute(input []byte) int {
r.SetUserValue("attr_val", attrVal)
hc.Handler().DownloadByAttribute(r)
return fuzzSuccessExitCode
}
func FuzzDownloadByAttribute(f *testing.F) {
@ -432,36 +416,36 @@ func InitFuzzHeadByAttribute() {
}
func DoFuzzHeadByAttribute(input []byte) int {
func DoFuzzHeadByAttribute(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp)
if err != nil {
return fuzzFailExitCode
return
}
cid := cnrID.EncodeToString()
cid, err = maybeFillRandom(tp, cid)
if err != nil {
return fuzzFailExitCode
return
}
attrKey, err = maybeFillRandom(tp, attrKey)
if err != nil {
return fuzzFailExitCode
return
}
attrVal, err = maybeFillRandom(tp, attrVal)
if err != nil {
return fuzzFailExitCode
return
}
r := new(fasthttp.RequestCtx)
@ -471,8 +455,6 @@ func DoFuzzHeadByAttribute(input []byte) int {
r.SetUserValue("attr_val", attrVal)
hc.Handler().HeadByAttribute(r)
return fuzzSuccessExitCode
}
func FuzzHeadByAttribute(f *testing.F) {
@ -485,32 +467,32 @@ func InitFuzzDownloadZipped() {
}
func DoFuzzDownloadZipped(input []byte) int {
func DoFuzzDownloadZipped(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
ctx, hc, cnrID, _, _, _, _, err := upload(tp)
if err != nil {
return fuzzFailExitCode
return
}
cid := cnrID.EncodeToString()
cid, err = maybeFillRandom(tp, cid)
if err != nil {
return fuzzFailExitCode
return
}
prefix := ""
prefix, err = maybeFillRandom(tp, prefix)
if err != nil {
return fuzzFailExitCode
return
}
r := new(fasthttp.RequestCtx)
@ -519,8 +501,6 @@ func DoFuzzDownloadZipped(input []byte) int {
r.SetUserValue("prefix", prefix)
hc.Handler().DownloadZip(r)
return fuzzSuccessExitCode
}
func FuzzDownloadZipped(f *testing.F) {
@ -533,21 +513,21 @@ func InitFuzzStoreBearerTokenAppCtx() {
}
func DoFuzzStoreBearerTokenAppCtx(input []byte) int {
func DoFuzzStoreBearerTokenAppCtx(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
prefix := ""
prefix, err = maybeFillRandom(tp, prefix)
if err != nil {
return fuzzFailExitCode
return
}
ctx := context.Background()
@ -570,8 +550,6 @@ func DoFuzzStoreBearerTokenAppCtx(input []byte) int {
}
tokens.StoreBearerTokenAppCtx(ctx, r)
return fuzzSuccessExitCode
}
func FuzzStoreBearerTokenAppCtx(f *testing.F) {

View file

@ -520,23 +520,15 @@ func TestIndex(t *testing.T) {
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
obj2ID := oidtest.ID()
obj2 := object.New()
obj2.SetID(obj2ID)
obj2.SetPayload([]byte("obj2"))
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/.."))
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
hc.tree.containers[cnrID.String()] = containerInfo{
trees: map[string]map[string]nodeResponse{
"system": {"bucket-settings": nodeResponse{nodeID: 1}},
"version": {
"<root>": nodeResponse{}, //root
"": nodeResponse{}, //root
"prefix": nodeResponse{
nodeID: 1,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}},
},
"prefix/obj1": nodeResponse{
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}},
"obj1": nodeResponse{
parentID: 1,
nodeID: 2,
meta: []nodeMeta{
@ -544,23 +536,6 @@ func TestIndex(t *testing.T) {
{key: "OID", value: []byte(obj1ID.String())},
},
},
"": nodeResponse{
nodeID: 3,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("")}},
},
"/dir": nodeResponse{
parentID: 3,
nodeID: 4,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("dir")}},
},
"/dir/..": nodeResponse{
parentID: 4,
nodeID: 5,
meta: []nodeMeta{
{key: tree.FileNameKey, value: []byte("..")},
{key: "OID", value: []byte(obj2ID.String())},
},
},
},
},
}
@ -588,21 +563,6 @@ func TestIndex(t *testing.T) {
r = prepareGetRequest(ctx, "bucket", "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy")
r = prepareGetRequest(ctx, "bucket", "")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//">/</a>`)
r = prepareGetRequest(ctx, "bucket", "/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//dir/">dir/</a>`)
r = prepareGetRequest(ctx, "bucket", "/dir/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//dir%2F..">..</a>`)
})
t.Run("native", func(t *testing.T) {
@ -615,13 +575,6 @@ func TestIndex(t *testing.T) {
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
obj2ID := oidtest.ID()
obj2 := object.New()
obj2.SetID(obj2ID)
obj2.SetPayload([]byte("obj2"))
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/.."))
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
@ -645,21 +598,6 @@ func TestIndex(t *testing.T) {
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy")
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//">/</a>`)
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//dir/">dir/</a>`)
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/dir/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/`+obj2ID.String()+`">..</a>`)
})
}

View file

@ -127,7 +127,6 @@ const (
EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header"
CORSRuleWasNotMatched = "cors rule was not matched"
CouldntCacheCors = "couldn't cache cors"
BrowseIndex = "browse index"
)
// Log messages with the "external_storage" tag.

View file

@ -56,24 +56,40 @@
</thead>
<tbody>
{{ $parentPrefix := getParent .Prefix }}
{{if $parentPrefix }}
<tr>
<td>
⮐<a href="/get/{{$container}}/{{ urlencode $parentPrefix }}">..</a>
⮐<a href="/get/{{$container}}{{ urlencode $parentPrefix }}/">..</a>
</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
{{else}}
<tr>
<td>
⮐<a href="/get/{{$container}}/">..</a>
</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
{{end}}
{{range .Objects}}
<tr>
<td>
{{if .IsDir}}
🗀
<a href="{{.GetURL}}/">{{.FileName}}/</a>
<a href="{{.GetURL}}/">
{{.FileName}}/
</a>
{{else}}
🗎
<a href="{{ .GetURL }}">{{.FileName}}</a>
<a href="{{ .GetURL }}">
{{.FileName}}
</a>
{{end}}
</td>
<td>{{.OID}}</td>

View file

@ -323,23 +323,17 @@ func pathFromName(objectName string) []string {
return strings.Split(objectName, separator)
}
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix *string, latestOnly bool) ([]data.NodeInfo, error) {
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix")
defer span.End()
rootID := []uint64{0}
var err error
if prefix != nil {
rootID, err = c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(*prefix, separator))
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil
}
return nil, err
rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator))
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil
}
return nil, err
}
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {