Compare commits

..

8 commits

Author SHA1 Message Date
Aleksey Kravchenko
ace4e91a18 [#222] Add support for aio v1.7.0 in integration tests
Signed-off-by: Aleksey Kravchenko <al.kravchenko@yadro.com>
2025-05-13 14:18:40 +00:00
381fbf0921 [#237] Improve request log
It's better to see original URI rather than decoded

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-05-13 13:55:24 +00:00
d85d7f319f [#240] Fix native index page
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-05-13 16:20:11 +03:00
ff8eb082d9 [#240] Fix s3 index page
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-05-13 16:20:09 +03:00
e78e143e38 [#226] Improve CORS validation
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2025-05-13 13:00:13 +00:00
50021ce587 [#236] Switch base image for OCI container to Debian
Related: TrueCloudLab/frostfs-node#1724

Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>
2025-05-12 15:09:19 +00:00
5e6ee3a41c [#241] Add fuzzing files removing to make clean
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2025-05-07 10:35:37 +03:00
3aed60aa79 [#241] Remove exit codes from fuzzing tests
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2025-05-07 10:35:26 +03:00
16 changed files with 439 additions and 121 deletions

View file

@ -1,7 +1,10 @@
FROM golang:1.24-alpine AS basebuilder
RUN apk add --update make bash ca-certificates
FROM basebuilder AS builder
FROM golang:1.24 AS builder
RUN apt-get update && \
apt-get install --no-install-recommends -y \
bash \
ca-certificates \
&& \
rm -rf /var/lib/apt/lists/*
ENV GOGC=off
ENV CGO_ENABLED=0
ARG BUILD=now
@ -13,7 +16,7 @@ COPY . /src
RUN make
# Executable image
FROM scratch
FROM debian:stable-slim
WORKDIR /

View file

@ -1,5 +1,10 @@
FROM alpine
RUN apk add --update --no-cache bash ca-certificates
FROM debian:stable-slim
RUN apt-get update && \
apt-get install --no-install-recommends -y \
bash \
ca-certificates \
&& \
rm -rf /var/lib/apt/lists/*
WORKDIR /

View file

@ -31,6 +31,10 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
.PHONY: debpackage debclean
FUZZING_DIR = $(shell pwd)/tests/fuzzing/files
FUZZING_TEMP_DIRS = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/tempfuzz ")
FUZZING_COVER_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/cover.out ")
FUZZING_FUNC_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/func.txt ")
FUZZING_INDEX_FILES = $(shell find . -type f -name '*_fuzz_test.go' -exec dirname {} \; | uniq | xargs -I{} echo -n "{}/index.html ")
NGFUZZ_REPO = https://gitflic.ru/project/yadro/ngfuzz.git
FUZZ_TIMEOUT ?= 30
FUZZ_FUNCTIONS ?= ""
@ -188,7 +192,10 @@ version:
# Clean up
clean:
rm -rf vendor
rm -rf $(BINDIR)
rm -rf $(BINDIR)
rm -rf $(FUZZING_DIR) $(FUZZING_TEMP_DIRS)
rm -f $(FUZZING_COVER_FILES) $(FUZZING_FUNC_FILES) $(FUZZING_INDEX_FILES)
git checkout -- go.mod go.sum
# Package for Debian
debpackage:

View file

@ -863,8 +863,7 @@ func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler {
log.Info(logs.Request, zap.String("remote", req.RemoteAddr().String()),
zap.ByteString("method", req.Method()),
zap.ByteString("path", req.Path()),
zap.ByteString("query", req.QueryArgs().QueryString()),
zap.ByteString("uri", req.RequestURI()),
logs.TagField(logs.TagDatapath))
h(req)
}

View file

@ -14,10 +14,12 @@ import (
"net/http"
"os"
"sort"
"strconv"
"strings"
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape"
containerv2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
@ -31,6 +33,8 @@ import (
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"git.frostfs.info/TrueCloudLab/policy-engine/schema/native"
docker "github.com/docker/docker/api/types/container"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/wallet"
@ -51,6 +55,59 @@ const (
testCORSContainerName = "cors"
)
func versionToUint(t *testing.T, v string) uint64 {
parts := strings.Split(v, ".")
require.Len(t, parts, 3)
major, err := strconv.ParseUint(parts[0], 10, 16)
require.NoError(t, err)
minor, err := strconv.ParseUint(parts[1], 10, 16)
require.NoError(t, err)
patch, err := strconv.ParseUint(parts[2], 10, 32)
require.NoError(t, err)
versionNumber := major<<48 | minor<<32 | patch
return versionNumber
}
func publicReadWriteRules() []chain.Rule {
return []chain.Rule{
{
Status: chain.Allow,
Actions: chain.Actions{
Inverted: false,
Names: []string{
native.MethodPutObject,
native.MethodGetObject,
native.MethodHeadObject,
native.MethodDeleteObject,
native.MethodSearchObject,
native.MethodRangeObject,
native.MethodHashObject,
native.MethodPatchObject,
},
},
Resources: chain.Resources{
Inverted: false,
Names: []string{native.ResourceFormatRootObjects},
},
Any: false,
},
}
}
func privateRules() []chain.Rule {
rule := publicReadWriteRules()
// The same as public-read-write, except that only the owner is allowed to perform the listed actions
rule[0].Condition = []chain.Condition{
{
Op: chain.CondStringEquals,
Kind: chain.KindRequest,
Key: native.PropertyKeyActorRole,
Value: native.PropertyValueContainerRoleOwner,
},
}
return rule
}
func TestIntegration(t *testing.T) {
rootCtx := context.Background()
aioImage := "git.frostfs.info/truecloudlab/frostfs-aio:"
@ -59,6 +116,7 @@ func TestIntegration(t *testing.T) {
"1.3.0",
"1.5.0",
"1.6.5",
"1.7.0",
}
key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb")
require.NoError(t, err)
@ -75,10 +133,16 @@ func TestIntegration(t *testing.T) {
ctx, cancel2 := context.WithCancel(rootCtx)
aioContainer := createDockerContainer(ctx, t, aioImage+version)
if strings.HasPrefix(version, "1.6") {
versionNumber := versionToUint(t, version)
if versionNumber >= versionToUint(t, "1.6.0") {
registerUser(t, ctx, aioContainer, file.Name())
}
createContainer := createContainerWithACL
if versionNumber >= versionToUint(t, "1.7.0") {
createContainer = createContainerWithAPE
}
// Creating CORS container
clientPool := getPool(ctx, t, key)
_, err = createContainer(ctx, t, clientPool, ownerID, testCORSContainerName)
@ -469,15 +533,18 @@ func checkStatusCodes(ctx context.Context, t *testing.T, clientPool *pool.Pool,
t.Run("access denied", func(t *testing.T) {
basicACL := acl.Private
var recs []*eacl.Record
var APERules []chain.Rule
if version == "1.2.7" {
basicACL = acl.PublicRWExtended
rec := eacl.NewRecord()
rec.SetAction(eacl.ActionDeny)
rec.SetOperation(eacl.OperationGet)
recs = append(recs, rec)
} else if versionToUint(t, version) >= versionToUint(t, "1.7.0") {
APERules = privateRules()
}
cnrID, err := createContainerBase(ctx, t, clientPool, ownerID, basicACL, "")
cnrID, err := createContainerBase(ctx, t, clientPool, ownerID, basicACL, APERules, "")
require.NoError(t, err)
key, err := keys.NewPrivateKey()
@ -560,11 +627,77 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool
return clientPool
}
func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, name string) (cid.ID, error) {
return createContainerBase(ctx, t, clientPool, ownerID, acl.PublicRWExtended, name)
func createContainerWithACL(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, name string) (cid.ID, error) {
return createContainerBase(ctx, t, clientPool, ownerID, acl.PublicRWExtended, nil, name)
}
func createContainerBase(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, basicACL acl.Basic, name string) (cid.ID, error) {
func createContainerWithAPE(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, name string) (cid.ID, error) {
return createContainerBase(ctx, t, clientPool, ownerID, 0, publicReadWriteRules(), name)
}
func waitForAPEBeApplied(ctx context.Context, clientPool *pool.Pool, expectedCh chain.Chain, cnrID cid.ID) error {
prmListAPEChains := pool.PrmListAPEChains{
Target: ape.ChainTarget{
TargetType: ape.TargetTypeContainer,
Name: cnrID.EncodeToString(),
},
}
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
for {
chains, err := clientPool.ListAPEChains(ctx, prmListAPEChains)
if err != nil {
return fmt.Errorf("list APE chains: %w", err)
}
for _, rawChain := range chains {
var ch chain.Chain
err = ch.UnmarshalBinary(rawChain.Raw)
if err != nil {
return fmt.Errorf("unmarshal chain: %w", err)
}
if bytes.Equal(ch.ID, expectedCh.ID) {
// At the moment, according to the core team, there is no way through the API
// to check whether the APE chain stored in the contract has been applied to the container.
// So after we make sure that the APE chain is stored, we just wait for a certain period of time
// (8 seconds by default, the time until the next block and cache invalidation)
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(8 * time.Second):
return nil
}
}
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(500 * time.Millisecond):
}
}
}
func addAPEChainToContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, chainID string, rules []chain.Rule, cnrID cid.ID) {
ch := chain.Chain{
ID: chain.ID(chainID),
Rules: rules,
}
data, err := ch.MarshalBinary()
require.NoError(t, err)
prmAddAPEChain := pool.PrmAddAPEChain{
Target: ape.ChainTarget{
TargetType: ape.TargetTypeContainer,
Name: cnrID.EncodeToString(),
},
Chain: ape.Chain{Raw: data},
}
err = clientPool.AddAPEChain(ctx, prmAddAPEChain)
require.NoError(t, err)
err = waitForAPEBeApplied(ctx, clientPool, ch, cnrID)
require.NoError(t, err)
}
func createContainerBase(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, basicACL acl.Basic, apeRules []chain.Rule, name string) (cid.ID, error) {
var policy netmap.PlacementPolicy
err := policy.DecodeString("REP 1")
require.NoError(t, err)
@ -572,7 +705,9 @@ func createContainerBase(ctx context.Context, t *testing.T, clientPool *pool.Poo
var cnr container.Container
cnr.Init()
cnr.SetPlacementPolicy(policy)
cnr.SetBasicACL(basicACL)
if basicACL != 0 {
cnr.SetBasicACL(basicACL)
}
cnr.SetOwner(ownerID)
container.SetCreationTime(&cnr, time.Now())
@ -601,6 +736,11 @@ func createContainerBase(ctx context.Context, t *testing.T, clientPool *pool.Poo
}
fmt.Println(CID.String())
if len(apeRules) != 0 {
chainID := "http-aio-" + CID.String()[:8]
addAPEChainToContainer(ctx, t, clientPool, chainID, apeRules, CID)
}
return CID, err
}

3
go.mod
View file

@ -8,6 +8,7 @@ require (
git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250425083815-09ff3bf14991
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
github.com/bluele/gcache v0.0.2
github.com/docker/docker v27.1.1+incompatible
@ -68,10 +69,12 @@ require (
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/ipfs/go-cid v0.0.7 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.17.4 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect

2
go.sum
View file

@ -52,6 +52,8 @@ git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8l
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8=
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972/go.mod h1:2hM42MBrlhvN6XToaW6OWNk5ZLcu1FhaukGgxtfpDDI=
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250425083815-09ff3bf14991 h1:eTefR8y2y9cg7X5kybIcXDdmABfk/3A2awdmFD3zOsA=
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250425083815-09ff3bf14991/go.mod h1:GZTk55RI4dKzsK6BCn5h2xxE28UHNfgoq/NJxW/LQ6A=
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA=
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc=
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA=

View file

@ -76,13 +76,13 @@ func newListObjectsResponseNative(attrs map[string]string) ResponseObject {
}
}
func getNextDir(filepath, prefix string) string {
func getNextDir(filepath, prefix string) *string {
restPath := strings.Replace(filepath, prefix, "", 1)
index := strings.Index(restPath, "/")
if index == -1 {
return ""
return nil
}
return restPath[:index]
return ptr(restPath[:index])
}
func lastPathElement(path string) string {
@ -143,15 +143,18 @@ func getParent(encPrefix string) string {
if slashIndex == -1 {
return ""
}
return prefix[:slashIndex]
return prefix[:slashIndex+1]
}
func urlencode(path string) string {
var res strings.Builder
prefixParts := strings.Split(path, "/")
for _, prefixPart := range prefixParts {
prefixPart = "/" + url.PathEscape(prefixPart)
for i, prefixPart := range prefixParts {
prefixPart = url.PathEscape(prefixPart)
if i != 0 {
prefixPart = "/" + prefixPart
}
if prefixPart == "/." || prefixPart == "/.." {
prefixPart = url.PathEscape(prefixPart)
}
@ -168,11 +171,16 @@ type GetObjectsResponse struct {
}
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
if prefix != "" && prefix[len(prefix)-1] == '/' {
prefix = prefix[:len(prefix)-1]
var treePrefix *string
if prefix != "" {
if prefix[len(prefix)-1] == '/' {
treePrefix = ptr(prefix[:len(prefix)-1])
} else {
treePrefix = &prefix
}
}
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, treePrefix, true)
if err != nil {
return nil, err
}
@ -193,14 +201,18 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
if obj.IsDeleteMarker {
continue
}
obj.FilePath = prefix + "/" + obj.FileName
obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath)
obj.FilePath = prefix + obj.FileName
obj.GetURL = "/get/" + bucketInfo.Name + "/" + urlencode(obj.FilePath)
result.objects = append(result.objects, obj)
}
return result, nil
}
func ptr(s string) *string {
return &s
}
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
basePath := prefix
if basePath != "" && basePath[len(basePath)-1] != '/' {
@ -247,7 +259,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck
if _, ok := dirs[objExt.Object.FileName]; ok {
continue
}
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + urlencode(objExt.Object.FilePath)
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + urlencode(objExt.Object.FilePath)
dirs[objExt.Object.FileName] = struct{}{}
} else {
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + objExt.Object.OID
@ -319,13 +331,13 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID,
}
dirname := getNextDir(attrs[object.AttributeFilePath], basePath)
if dirname == "" {
if dirname == nil {
return newListObjectsResponseNative(attrs), nil
}
return ResponseObject{
FileName: dirname,
FilePath: basePath + dirname,
FileName: *dirname,
FilePath: basePath + *dirname,
IsDir: true,
}, nil
}

View file

@ -79,6 +79,10 @@ func (h *Handler) Preflight(req *fasthttp.RequestCtx) {
}
for _, rule := range corsConfig.CORSRules {
if err = checkCORSRuleWildcards(rule); err != nil {
log.Error(logs.InvalidCorsRule, zap.Error(err), logs.TagField(logs.TagDatapath))
continue
}
for _, o := range rule.AllowedOrigins {
if o == string(origin) || o == wildcard || (strings.Contains(o, "*") && match(o, string(origin))) {
for _, m := range rule.AllowedMethods {
@ -147,6 +151,10 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) {
}
for _, rule := range corsConfig.CORSRules {
if err = checkCORSRuleWildcards(rule); err != nil {
log.Error(logs.InvalidCorsRule, zap.Error(err), logs.TagField(logs.TagDatapath))
continue
}
for _, o := range rule.AllowedOrigins {
if o == string(origin) || (strings.Contains(o, "*") && len(o) > 1 && match(o, string(origin))) {
for _, m := range rule.AllowedMethods {
@ -178,6 +186,22 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) {
}
}
func checkCORSRuleWildcards(rule data.CORSRule) error {
for _, origin := range rule.AllowedOrigins {
if strings.Count(origin, wildcard) > 1 {
return fmt.Errorf("invalid allowed origin: %s", origin)
}
}
for _, header := range rule.AllowedHeaders {
if strings.Count(header, wildcard) > 1 {
return fmt.Errorf("invalid allowed header: %s", header)
}
}
return nil
}
func (h *Handler) getCORSConfig(ctx context.Context, log *zap.Logger, cidStr string) (*data.CORSConfiguration, error) {
cnrID, err := h.resolveContainer(ctx, cidStr)
if err != nil {

View file

@ -176,6 +176,55 @@ func TestPreflight(t *testing.T) {
},
status: fasthttp.StatusBadRequest,
},
{
name: "invalid allowed origin",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*.example.*"},
AllowedMethods: []string{"HEAD"},
},
},
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://www.example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderAccessControlAllowHeaders: "",
fasthttp.HeaderAccessControlExposeHeaders: "",
fasthttp.HeaderAccessControlMaxAge: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
status: fasthttp.StatusForbidden,
},
{
name: "invalid allowed header",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*example.com"},
AllowedMethods: []string{"HEAD"},
AllowedHeaders: []string{"x-amz-*-*"},
},
},
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://www.example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderAccessControlAllowHeaders: "",
fasthttp.HeaderAccessControlExposeHeaders: "",
fasthttp.HeaderAccessControlMaxAge: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
status: fasthttp.StatusForbidden,
},
} {
t.Run(tc.name, func(t *testing.T) {
if tc.corsConfig != nil {
@ -328,6 +377,47 @@ func TestSetCORSHeaders(t *testing.T) {
fasthttp.HeaderAccessControlAllowCredentials: "",
},
},
{
name: "invalid allowed origin",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*.example.*"},
AllowedMethods: []string{"GET"},
},
},
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://www.example.com",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderVary: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
},
{
name: "invalid allowed header",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*example.com"},
AllowedMethods: []string{"GET"},
AllowedHeaders: []string{"x-amz-*-*"},
},
},
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://www.example.com",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderVary: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
},
} {
t.Run(tc.name, func(t *testing.T) {
epoch++

View file

@ -325,11 +325,12 @@ func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex")
defer span.End()
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
h.reqLogger(ctx).Info(logs.BrowseIndex,
zap.String("bucket", prm.BktInfo.Name),
zap.String("container", prm.BktInfo.CID.EncodeToString()),
zap.String("prefix", prm.Path),
))
logs.TagField(logs.TagDatapath),
)
objects, err := fn(ctx, prm.BktInfo, prm.Path)
if err != nil {

View file

@ -24,11 +24,6 @@ import (
"go.uber.org/zap"
)
const (
fuzzSuccessExitCode = 0
fuzzFailExitCode = -1
)
func prepareStrings(tp *go_fuzz_utils.TypeProvider, count int) ([]string, error) {
array := make([]string, count)
var err error
@ -222,23 +217,18 @@ func InitFuzzUpload() {
}
func DoFuzzUpload(input []byte) int {
func DoFuzzUpload(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
_, _, _, _, _, _, _, err = upload(tp)
if err != nil {
return fuzzFailExitCode
}
return fuzzSuccessExitCode
_, _, _, _, _, _, _, _ = upload(tp)
}
func FuzzUpload(f *testing.F) {
@ -307,30 +297,28 @@ func InitFuzzGet() {
}
func DoFuzzGet(input []byte) int {
func DoFuzzGet(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
ctx, hc, cnrID, resp, filename, _, _, err := upload(tp)
if err != nil {
return fuzzFailExitCode
return
}
r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename)
if err != nil {
return fuzzFailExitCode
return
}
hc.Handler().DownloadByAddressOrBucketName(r)
return fuzzSuccessExitCode
}
func FuzzGet(f *testing.F) {
@ -343,30 +331,28 @@ func InitFuzzHead() {
}
func DoFuzzHead(input []byte) int {
func DoFuzzHead(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
ctx, hc, cnrID, resp, filename, _, _, err := upload(tp)
if err != nil {
return fuzzFailExitCode
return
}
r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename)
if err != nil {
return fuzzFailExitCode
return
}
hc.Handler().HeadByAddressOrBucketName(r)
return fuzzSuccessExitCode
}
func FuzzHead(f *testing.F) {
@ -379,36 +365,36 @@ func InitFuzzDownloadByAttribute() {
}
func DoFuzzDownloadByAttribute(input []byte) int {
func DoFuzzDownloadByAttribute(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp)
if err != nil {
return fuzzFailExitCode
return
}
cid := cnrID.EncodeToString()
cid, err = maybeFillRandom(tp, cid)
if err != nil {
return fuzzFailExitCode
return
}
attrKey, err = maybeFillRandom(tp, attrKey)
if err != nil {
return fuzzFailExitCode
return
}
attrVal, err = maybeFillRandom(tp, attrVal)
if err != nil {
return fuzzFailExitCode
return
}
r := new(fasthttp.RequestCtx)
@ -418,8 +404,6 @@ func DoFuzzDownloadByAttribute(input []byte) int {
r.SetUserValue("attr_val", attrVal)
hc.Handler().DownloadByAttribute(r)
return fuzzSuccessExitCode
}
func FuzzDownloadByAttribute(f *testing.F) {
@ -432,36 +416,36 @@ func InitFuzzHeadByAttribute() {
}
func DoFuzzHeadByAttribute(input []byte) int {
func DoFuzzHeadByAttribute(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp)
if err != nil {
return fuzzFailExitCode
return
}
cid := cnrID.EncodeToString()
cid, err = maybeFillRandom(tp, cid)
if err != nil {
return fuzzFailExitCode
return
}
attrKey, err = maybeFillRandom(tp, attrKey)
if err != nil {
return fuzzFailExitCode
return
}
attrVal, err = maybeFillRandom(tp, attrVal)
if err != nil {
return fuzzFailExitCode
return
}
r := new(fasthttp.RequestCtx)
@ -471,8 +455,6 @@ func DoFuzzHeadByAttribute(input []byte) int {
r.SetUserValue("attr_val", attrVal)
hc.Handler().HeadByAttribute(r)
return fuzzSuccessExitCode
}
func FuzzHeadByAttribute(f *testing.F) {
@ -485,32 +467,32 @@ func InitFuzzDownloadZipped() {
}
func DoFuzzDownloadZipped(input []byte) int {
func DoFuzzDownloadZipped(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
ctx, hc, cnrID, _, _, _, _, err := upload(tp)
if err != nil {
return fuzzFailExitCode
return
}
cid := cnrID.EncodeToString()
cid, err = maybeFillRandom(tp, cid)
if err != nil {
return fuzzFailExitCode
return
}
prefix := ""
prefix, err = maybeFillRandom(tp, prefix)
if err != nil {
return fuzzFailExitCode
return
}
r := new(fasthttp.RequestCtx)
@ -519,8 +501,6 @@ func DoFuzzDownloadZipped(input []byte) int {
r.SetUserValue("prefix", prefix)
hc.Handler().DownloadZip(r)
return fuzzSuccessExitCode
}
func FuzzDownloadZipped(f *testing.F) {
@ -533,21 +513,21 @@ func InitFuzzStoreBearerTokenAppCtx() {
}
func DoFuzzStoreBearerTokenAppCtx(input []byte) int {
func DoFuzzStoreBearerTokenAppCtx(input []byte) {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
return
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
return
}
prefix := ""
prefix, err = maybeFillRandom(tp, prefix)
if err != nil {
return fuzzFailExitCode
return
}
ctx := context.Background()
@ -570,8 +550,6 @@ func DoFuzzStoreBearerTokenAppCtx(input []byte) int {
}
tokens.StoreBearerTokenAppCtx(ctx, r)
return fuzzSuccessExitCode
}
func FuzzStoreBearerTokenAppCtx(f *testing.F) {

View file

@ -520,15 +520,23 @@ func TestIndex(t *testing.T) {
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
obj2ID := oidtest.ID()
obj2 := object.New()
obj2.SetID(obj2ID)
obj2.SetPayload([]byte("obj2"))
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/.."))
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
hc.tree.containers[cnrID.String()] = containerInfo{
trees: map[string]map[string]nodeResponse{
"system": {"bucket-settings": nodeResponse{nodeID: 1}},
"version": {
"": nodeResponse{}, //root
"<root>": nodeResponse{}, //root
"prefix": nodeResponse{
nodeID: 1,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}},
"obj1": nodeResponse{
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}},
},
"prefix/obj1": nodeResponse{
parentID: 1,
nodeID: 2,
meta: []nodeMeta{
@ -536,6 +544,23 @@ func TestIndex(t *testing.T) {
{key: "OID", value: []byte(obj1ID.String())},
},
},
"": nodeResponse{
nodeID: 3,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("")}},
},
"/dir": nodeResponse{
parentID: 3,
nodeID: 4,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("dir")}},
},
"/dir/..": nodeResponse{
parentID: 4,
nodeID: 5,
meta: []nodeMeta{
{key: tree.FileNameKey, value: []byte("..")},
{key: "OID", value: []byte(obj2ID.String())},
},
},
},
},
}
@ -563,6 +588,21 @@ func TestIndex(t *testing.T) {
r = prepareGetRequest(ctx, "bucket", "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy")
r = prepareGetRequest(ctx, "bucket", "")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//">/</a>`)
r = prepareGetRequest(ctx, "bucket", "/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//dir/">dir/</a>`)
r = prepareGetRequest(ctx, "bucket", "/dir/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//dir%2F..">..</a>`)
})
t.Run("native", func(t *testing.T) {
@ -575,6 +615,13 @@ func TestIndex(t *testing.T) {
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
obj2ID := oidtest.ID()
obj2 := object.New()
obj2.SetID(obj2ID)
obj2.SetPayload([]byte("obj2"))
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/.."))
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
@ -598,6 +645,21 @@ func TestIndex(t *testing.T) {
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy")
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//">/</a>`)
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//dir/">dir/</a>`)
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/dir/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/`+obj2ID.String()+`">..</a>`)
})
}

View file

@ -127,6 +127,8 @@ const (
EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header"
CORSRuleWasNotMatched = "cors rule was not matched"
CouldntCacheCors = "couldn't cache cors"
InvalidCorsRule = "invalid cors rule, skip"
BrowseIndex = "browse index"
)
// Log messages with the "external_storage" tag.

View file

@ -56,40 +56,24 @@
</thead>
<tbody>
{{ $parentPrefix := getParent .Prefix }}
{{if $parentPrefix }}
<tr>
<td>
⮐<a href="/get/{{$container}}{{ urlencode $parentPrefix }}/">..</a>
⮐<a href="/get/{{$container}}/{{ urlencode $parentPrefix }}">..</a>
</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
{{else}}
<tr>
<td>
⮐<a href="/get/{{$container}}/">..</a>
</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
{{end}}
{{range .Objects}}
<tr>
<td>
{{if .IsDir}}
🗀
<a href="{{.GetURL}}/">
{{.FileName}}/
</a>
<a href="{{.GetURL}}/">{{.FileName}}/</a>
{{else}}
🗎
<a href="{{ .GetURL }}">
{{.FileName}}
</a>
<a href="{{ .GetURL }}">{{.FileName}}</a>
{{end}}
</td>
<td>{{.OID}}</td>

View file

@ -323,17 +323,23 @@ func pathFromName(objectName string) []string {
return strings.Split(objectName, separator)
}
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) {
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix *string, latestOnly bool) ([]data.NodeInfo, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix")
defer span.End()
rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator))
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil
rootID := []uint64{0}
var err error
if prefix != nil {
rootID, err = c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(*prefix, separator))
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil
}
return nil, err
}
return nil, err
}
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {