ognev/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Pull requests Activity

[#480] Add fuzzing tests

Browse source 
Signed-off-by: Roman Ognev <r.ognev@yadro.com>
This commit is contained in:
Roman Ognev 2024-06-17 14:36:53 +03:00
parent 136b5521fe
commit 3bc699bd11
9 changed files with 1233 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
Makefile
View file

@ -23,6 +23,15 @@ OUTPUT_LINT_DIR ?= $(shell pwd)/bin
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
TMP_DIR := .cache
# Variables for fuzzing
FUZZING_DIR = $(shell pwd)/tests/fuzzing/files
GO_118_FUZZ_BUILD_REPO = b.yadro.com/obj/go-118-fuzz-build
FUZZ_NGFUZZ_DIR ?= ""
FUZZ_TIMEOUT ?= 30
FUZZ_FUNCTIONS ?= "all"
FUZZ_AUX ?= ""
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
# .deb package versioning
@ -76,6 +85,38 @@ cover:
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
@go tool cover -html=coverage.txt -o coverage.html
# Run fuzzing
CLANG := $(shell which clang-17 2>/dev/null)
.PHONY: check-clang all
check-clang:
ifeq ($(CLANG),)
@echo "clang-17 is not installed. Please install it before proceeding - https://apt.llvm.org/llvm.sh "
@exit 1
endif
.PHONY: check-ngfuzz all
check-ngfuzz:
@if [ -z "$(FUZZ_NGFUZZ_DIR)" ]; then \
echo "Please set a variable FUZZ_NGFUZZ_DIR to specify path to the ngfuzz"; \
exit 1; \
fi
.PHONY: install-fuzzing-deps
install-fuzzing-deps: check-clang check-ngfuzz
ifeq (,$(wildcard $(FUZZING_DIR)/go-118-fuzz-build))
GOBIN=$(FUZZING_DIR) go install $(GO_118_FUZZ_BUILD_REPO)
endif
.PHONY: fuzz
fuzz: install-fuzzing-deps
@START_PATH=$$(pwd); \
ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \
cd $(FUZZ_NGFUZZ_DIR) && \
./ngfuzz -clean && \
./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \
./ngfuzz -report
# Reformat code
format:
@echo "⇒ Processing gofmt check"
@ -148,6 +189,7 @@ version:
clean:
rm -rf .cache
rm -rf $(BINDIR)
rm -rf $(FUZZING_DIR)
# Generate code from .proto files
protoc:

17
README.md
View file

@ -93,6 +93,23 @@ HTTP/1.1 200 OK
Also, you can configure domains using `.env` variables or `yaml` file.
## Fuzzing
To run fuzzing tests use the following command:
```shell
$ make fuzz
```
This command will install dependencies for the fuzzing process and run existing fuzzing tests.
You can also use the following arguments:
```
FUZZ_TIMEOUT - time to run each fuzzing test (default 30)
FUZZ_FUNCTIONS - fuzzing tests that will be started (default "all")
FUZZ_AUX - additional parameters for the fuzzer (for example, "-debug")
````
## Documentation
- [Configuration](./docs/configuration.md)

88
api/auth/center_fuzz_test.go Normal file
View file

@ -0,0 +1,88 @@
//go:build gofuzz
// +build gofuzz
package auth
import (
"strings"
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/aws/aws-sdk-go/aws/credentials"
go_fuzz_utils "github.com/trailofbits/go-fuzz-utils"
)
const (
fuzzSuccessExitCode = 1
fuzzFailExitCode = 0
)
func InitFuzzAuthenticate() {
}
func DoFuzzAuthenticate(input []byte) int {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
}
tp, err := go_fuzz_utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
}
var accessKeyAddr oid.Address
err = tp.Fill(accessKeyAddr)
if err != nil {
return fuzzFailExitCode
}
accessKeyID := strings.ReplaceAll(accessKeyAddr.String(), "/", "0")
secretKey, err := tp.GetString()
awsCreds := credentials.NewStaticCredentials(accessKeyID, secretKey, "")
reqData := RequestData{
Method: "GET",
Endpoint: "http://localhost:8084",
Bucket: "my-bucket",
Object: "@obj/name",
}
presignData := PresignData{
Service: "s3",
Region: "spb",
Lifetime: 10 * time.Minute,
SignTime: time.Now().UTC(),
}
req, err := PresignRequest(awsCreds, reqData, presignData)
if req == nil {
return fuzzFailExitCode
}
expBox := &accessbox.Box{
Gate: &accessbox.GateData{
SecretKey: secretKey,
},
}
mock := newTokensFrostfsMock()
mock.addBox(accessKeyAddr, expBox)
c := &Center{
cli: mock,
reg: NewRegexpMatcher(authorizationFieldRegexp),
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
}
c.Authenticate(req)
return fuzzSuccessExitCode
}
func FuzzAuthenticate(f *testing.F) {
f.Fuzz(func(t *testing.T, data []byte) {
DoFuzzAuthenticate(data)
})
}

1021
api/handler/handler_fuzz_test.go Normal file
View file

File diff suppressed because it is too large Load diff

59
api/handler/handlers_test.go
View file

@ -33,11 +33,15 @@ import (
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
"golang.org/x/exp/slices"
)
type handlerContext struct {
*handlerContextBase
t *testing.T
}
type handlerContextBase struct {
owner user.ID
t *testing.T
h *handler
@ -51,19 +55,19 @@ type handlerContext struct {
cache *layer.Cache
}
func (hc *handlerContext) Handler() *handler {
func (hc *handlerContextBase) Handler() *handler {
return hc.h
}
func (hc *handlerContext) MockedPool() *layer.TestFrostFS {
func (hc *handlerContextBase) MockedPool() *layer.TestFrostFS {
return hc.tp
}
func (hc *handlerContext) Layer() *layer.Layer {
func (hc *handlerContextBase) Layer() *layer.Layer {
return hc.h.obj
}
func (hc *handlerContext) Context() context.Context {
func (hc *handlerContextBase) Context() context.Context {
return hc.context
}
@ -137,19 +141,30 @@ func (c *configMock) RetryStrategy() RetryStrategy {
}
func prepareHandlerContext(t *testing.T) *handlerContext {
log := zaptest.NewLogger(t)
return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(log), log)
hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(zap.NewExample()))
require.NoError(t, err)
return &handlerContext{
handlerContextBase: hc,
t: t,
}
}
func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
log := zaptest.NewLogger(t)
return prepareHandlerContextBase(t, getMinCacheConfig(log), log)
hc, err := prepareHandlerContextBase(getMinCacheConfig(zap.NewExample()))
require.NoError(t, err)
return &handlerContext{
handlerContextBase: hc,
t: t,
}
}
func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig, log *zap.Logger) *handlerContext {
func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBase, error) {
key, err := keys.NewPrivateKey()
require.NoError(t, err)
if err != nil {
return nil, err
}
log := zap.NewExample()
tp := layer.NewTestFrostFS(key)
testResolver := &resolver.Resolver{Name: "test_resolver"}
@ -161,9 +176,11 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig, log *
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
memCli, err := tree.NewTreeServiceClientMemory()
require.NoError(t, err)
if err != nil {
return nil, err
}
treeMock := tree.NewTree(memCli, log)
treeMock := tree.NewTree(memCli, zap.NewExample())
features := &layer.FeatureSettingsMock{}
@ -178,7 +195,9 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig, log *
var pp netmap.PlacementPolicy
err = pp.DecodeString("REP 1")
require.NoError(t, err)
if err != nil {
return nil, err
}
cfg := &configMock{
defaultPolicy: pp,
@ -191,19 +210,23 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig, log *
frostfsid: newFrostfsIDMock(),
}
return &handlerContext{
accessBox, err := newTestAccessBox(key)
if err != nil {
return nil, err
}
return &handlerContextBase{
owner: owner,
t: t,
h: h,
tp: tp,
tree: treeMock,
context: middleware.SetBox(context.Background(), &middleware.Box{AccessBox: newTestAccessBox(t, key)}),
context: middleware.SetBox(context.Background(), &middleware.Box{AccessBox: accessBox}),
config: cfg,
layerFeatures: features,
treeMock: memCli,
cache: layerCfg.Cache,
}
}, nil
}
func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig {

12
api/handler/head_test.go
View file

@ -119,21 +119,25 @@ func TestIsAvailableToResolve(t *testing.T) {
}
}
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
func newTestAccessBox(key *keys.PrivateKey) (*accessbox.Box, error) {
var err error
if key == nil {
key, err = keys.NewPrivateKey()
require.NoError(t, err)
if err != nil {
return nil, err
}
}
var btoken bearer.Token
btoken.SetImpersonate(true)
err = btoken.Sign(key.PrivateKey)
require.NoError(t, err)
if err != nil {
return nil, err
}
return &accessbox.Box{
Gate: &accessbox.GateData{
BearerToken: &btoken,
},
}
}, nil
}

15
api/handler/object_list_test.go
View file

@ -100,7 +100,13 @@ func TestListObjectsWithOldTreeNodes(t *testing.T) {
func TestListObjectsVersionsSkipLogTaggingNodesError(t *testing.T) {
loggerCore, observedLog := observer.New(zap.DebugLevel)
log := zap.New(loggerCore)
hc := prepareHandlerContextBase(t, layer.DefaultCachesConfigs(log), log)
hcBase, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(log))
require.NoError(t, err)
hc := &handlerContext{
handlerContextBase: hcBase,
t: t,
}
bktName, objName := "bucket-versioning-enabled", "versions/object"
bktInfo := createTestBucket(hc, bktName)
@ -168,7 +174,12 @@ func TestListObjectsContextCanceled(t *testing.T) {
layerCfg.SessionList.Lifetime = time.Hour
layerCfg.SessionList.Size = 1
hc := prepareHandlerContextBase(t, layerCfg, log)
hcBase, err := prepareHandlerContextBase(layerCfg)
require.NoError(t, err)
hc := &handlerContext{
handlerContextBase: hcBase,
t: t,
}
bktName := "bucket-versioning-enabled"
bktInfo := createTestBucket(hc, bktName)

1
go.mod
View file

@ -25,6 +25,7 @@ require (
github.com/spf13/viper v1.15.0
github.com/ssgreg/journald v1.0.0
github.com/stretchr/testify v1.9.0
github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4
github.com/urfave/cli/v2 v2.3.0
go.opentelemetry.io/otel v1.16.0
go.opentelemetry.io/otel/trace v1.16.0

2
go.sum
View file

@ -321,6 +321,8 @@ github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8
github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs=
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM=
github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 h1:GpfJ7OdNjS7BFTVwNCUI9L4aCJOFRbr5fdHqjdhoYE8=
github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4/go.mod h1:f3jBhpWvuZmue0HZK52GzRHJOYHYSILs/c8+K2S/J+o=
github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=