[#135] Add fuzzing tests for handlers

Signed-off-by: Roman Ognev <r.ognev@yadro.com>

Makefile

@@ -30,6 +30,12 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
 			sed "s/-/~/")-${OS_RELEASE}
 .PHONY: debpackage debclean
 
+FUZZING_DIR = $(shell pwd)/tests/fuzzing/files
+FUZZ_NGFUZZ_DIR ?= ""
+FUZZ_TIMEOUT ?= 30
+FUZZ_FUNCTIONS ?= "all"
+FUZZ_AUX ?= ""
+
 # Make all binaries
 all: $(BINS)
 $(BINS):  $(DIRS) dep
@@ -78,6 +84,35 @@ cover:
 	@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
 	@go tool cover -html=coverage.txt -o coverage.html
 
+# Run fuzzing
+CLANG := $(shell which clang-17 2>/dev/null)
+.PHONY: check-clang all
+check-clang:
+ifeq ($(CLANG),)
+	@echo "clang-17 is not installed. Please install it before proceeding - https://apt.llvm.org/llvm.sh "
+	@exit 1
+endif
+
+.PHONY: check-ngfuzz all
+check-ngfuzz:
+	@if [ -z "$(FUZZ_NGFUZZ_DIR)" ]; then \
+		echo "Please set a variable FUZZ_NGFUZZ_DIR to specify path to the ngfuzz"; \
+		exit 1; \
+	fi
+
+.PHONY: install-fuzzing-deps
+install-fuzzing-deps: check-clang check-ngfuzz
+
+.PHONY: fuzz
+fuzz: install-fuzzing-deps  
+	@START_PATH=$$(pwd); \
+	ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \
+	cd $(FUZZ_NGFUZZ_DIR) && \
+	./ngfuzz -clean && \
+	./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \
+	./ngfuzz -report
+
+
 # Reformat code
 fmt:
 	@echo "⇒ Processing gofmt check"
@@ -150,6 +185,7 @@ version:
 clean:
 	rm -rf vendor
 	rm -rf $(BINDIR) 
+	rm -rf $(FUZZING_DIR)
 
 # Package for Debian
 debpackage:

README.md

@@ -575,3 +575,26 @@ See [configuration](./docs/gate-configuration.md).
 ## Credits
 
 Please see [CREDITS](CREDITS.md) for details.
+
+## Fuzzing
+
+To run fuzzing tests use the following command:
+
+```shell
+$ make fuzz
+```
+
+This command will install dependencies for the fuzzing process and run existing fuzzing tests.
+
+You can also use the following arguments:
+
+```
+FUZZ_TIMEOUT - time to run each fuzzing test (default 30) 
+FUZZ_FUNCTIONS - fuzzing tests that will be started (default "all")
+FUZZ_AUX - additional parameters for the fuzzer (for example, "-debug")
+FUZZ_NGFUZZ_DIR - path to ngfuzz tool
+````
+
+## Credits
+
+Please see [CREDITS](CREDITS.md) for details.

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/ssgreg/journald v1.0.0
 	github.com/stretchr/testify v1.9.0
 	github.com/testcontainers/testcontainers-go v0.13.0
+	github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4
 	github.com/valyala/fasthttp v1.34.0
 	go.opentelemetry.io/otel v1.16.0
 	go.opentelemetry.io/otel/trace v1.16.0

go.sum

@@ -820,6 +820,8 @@ github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBN
 github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 h1:GpfJ7OdNjS7BFTVwNCUI9L4aCJOFRbr5fdHqjdhoYE8=
+github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4/go.mod h1:f3jBhpWvuZmue0HZK52GzRHJOYHYSILs/c8+K2S/J+o=
 github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
 github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=

internal/handler/handler_fuzz_test.go

@@ -0,0 +1,574 @@
+//go:build gofuzz
+// +build gofuzz
+
+package handler
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"testing"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
+	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
+	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
+	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
+	go_fuzz_utils "github.com/trailofbits/go-fuzz-utils"
+	"github.com/valyala/fasthttp"
+)
+
+const (
+	fuzzSuccessExitCode = 0
+	fuzzFailExitCode    = -1
+)
+
+func prepareStrings(tp *go_fuzz_utils.TypeProvider, count int) ([]string, error) {
+	array := make([]string, count)
+	var err error
+
+	for i := 0; i < count; i++ {
+		err = tp.Reset()
+		if err != nil {
+			return nil, err
+		}
+
+		array[i], err = tp.GetString()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return array, nil
+}
+
+func prepareBools(tp *go_fuzz_utils.TypeProvider, count int) ([]bool, error) {
+	array := make([]bool, count)
+	var err error
+
+	for i := 0; i < count; i++ {
+		err = tp.Reset()
+		if err != nil {
+			return nil, err
+		}
+
+		array[i], err = tp.GetBool()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return array, nil
+}
+
+func getRandomDeterministicPositiveIntInRange(tp *go_fuzz_utils.TypeProvider, max int) (int, error) {
+	count, err := tp.GetInt()
+	if err != nil {
+		return -1, err
+	}
+	count = count % max
+	if count < 0 {
+		count += max
+	}
+	return count, nil
+}
+
+func generateHeaders(tp *go_fuzz_utils.TypeProvider, r *fasthttp.Request, params []string) error {
+	count, err := tp.GetInt()
+	if err != nil {
+		return err
+	}
+	count = count % len(params)
+	if count < 0 {
+		count += len(params)
+	}
+
+	for i := 0; i < count; i++ {
+		position, err := tp.GetInt()
+		if err != nil {
+			return err
+		}
+		position = position % len(params)
+		if position < 0 {
+			position += len(params)
+		}
+
+		v, err := tp.GetString()
+		if err != nil {
+			return err
+		}
+
+		r.Header.Set(params[position], v)
+
+	}
+
+	return nil
+}
+
+func maybeFillRandom(tp *go_fuzz_utils.TypeProvider, initValue string) (string, error) {
+	rnd, err := tp.GetBool()
+	if err != nil {
+		return "", err
+	}
+	if rnd == true {
+		initValue, err = tp.GetString()
+		if err != nil {
+			return "", err
+		}
+	}
+	return initValue, nil
+}
+
+func upload(tp *go_fuzz_utils.TypeProvider) (context.Context, *handlerContext, cid.ID, *fasthttp.RequestCtx, string, string, string, error) {
+	hc, err := prepareHandlerContext()
+	if err != nil {
+		return nil, nil, cid.ID{}, nil, "", "", "", err
+	}
+
+	aclList := []acl.Basic{
+		acl.Private,
+		acl.PrivateExtended,
+		acl.PublicRO,
+		acl.PublicROExtended,
+		acl.PublicRW,
+		acl.PublicRWExtended,
+		acl.PublicAppend,
+		acl.PublicAppendExtended,
+	}
+
+	pos, err := getRandomDeterministicPositiveIntInRange(tp, len(aclList))
+	if err != nil {
+		return nil, nil, cid.ID{}, nil, "", "", "", err
+	}
+	acl := aclList[pos]
+
+	strings, err := prepareStrings(tp, 6)
+	if err != nil {
+		return nil, nil, cid.ID{}, nil, "", "", "", err
+	}
+	bktName := strings[0]
+	objFileName := strings[1]
+	valAttr := strings[2]
+	keyAttr := strings[3]
+
+	if len(bktName) == 0 {
+		return nil, nil, cid.ID{}, nil, "", "", "", errors.New("not enought buckets")
+	}
+
+	cnrID, cnr, err := hc.prepareContainer(bktName, acl)
+	if err != nil {
+		return nil, nil, cid.ID{}, nil, "", "", "", err
+	}
+
+	hc.frostfs.SetContainer(cnrID, cnr)
+
+	ctx := context.Background()
+	ctx = middleware.SetNamespace(ctx, "")
+
+	r := new(fasthttp.RequestCtx)
+	utils.SetContextToRequest(ctx, r)
+	r.SetUserValue("cid", cnrID.EncodeToString())
+
+	attributes := map[string]string{
+		object.AttributeFileName: objFileName,
+		keyAttr:                  valAttr,
+	}
+
+	var buff bytes.Buffer
+	w := multipart.NewWriter(&buff)
+	fw, err := w.CreateFormFile("file", attributes[object.AttributeFileName])
+	if err != nil {
+		return nil, nil, cid.ID{}, nil, "", "", "", err
+	}
+
+	content, err := tp.GetBytes()
+	if err != nil {
+		return nil, nil, cid.ID{}, nil, "", "", "", err
+	}
+
+	if _, err = io.Copy(fw, bytes.NewReader(content)); err != nil {
+		return nil, nil, cid.ID{}, nil, "", "", "", err
+	}
+
+	if err = w.Close(); err != nil {
+		return nil, nil, cid.ID{}, nil, "", "", "", err
+	}
+
+	r.Request.SetBodyStream(&buff, buff.Len())
+	r.Request.Header.Set("Content-Type", w.FormDataContentType())
+	r.Request.Header.Set("X-Attribute-"+keyAttr, valAttr)
+
+	err = generateHeaders(tp, &r.Request, []string{"X-Attribute-", "X-Attribute-DupKey", "X-Attribute-MyAttribute", "X-Attribute-System-DupKey", "X-Attribute-System-Expiration-Epoch1", "X-Attribute-SYSTEM-Expiration-Epoch2", "X-Attribute-system-Expiration-Epoch3", "X-Attribute-User-Attribute", "X-Attribute-", "X-Attribute-FileName", "X-Attribute-FROSTFS", "X-Attribute-neofs", "X-Attribute-SYSTEM", "X-Attribute-System-Expiration-Duration", "X-Attribute-System-Expiration-Epoch", "X-Attribute-System-Expiration-RFC3339", "X-Attribute-System-Expiration-Timestamp", "X-Attribute-Timestamp", "X-Attribute-" + strings[4], "X-Attribute-System-" + strings[5]})
+	if err != nil {
+		return nil, nil, cid.ID{}, nil, "", "", "", err
+	}
+
+	hc.Handler().Upload(r)
+
+	if r.Response.StatusCode() != http.StatusOK {
+		return nil, nil, cid.ID{}, nil, "", "", "", errors.New("error on upload")
+	}
+
+	return ctx, hc, cnrID, r, objFileName, keyAttr, valAttr, nil
+}
+
+func InitFuzzUpload() {
+
+}
+
+func DoFuzzUpload(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := go_fuzz_utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	_, _, _, _, _, _, _, err = upload(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzUpload(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzUpload(data)
+	})
+}
+
+func downloadOrHead(tp *go_fuzz_utils.TypeProvider, ctx context.Context, hc *handlerContext, cnrID cid.ID, resp *fasthttp.RequestCtx, filename string) (*fasthttp.RequestCtx, error) {
+
+	var putRes putResponse
+
+	defer func() {
+		if r := recover(); r != nil {
+			panic(resp)
+		}
+	}()
+
+	data := resp.Response.Body()
+	err := json.Unmarshal(data, &putRes)
+
+	if err != nil {
+		return nil, err
+	}
+
+	obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
+	attr := object.NewAttribute()
+	attr.SetKey(object.AttributeFilePath)
+
+	filename, err = maybeFillRandom(tp, filename)
+	if err != nil {
+		return nil, err
+	}
+
+	attr.SetValue(filename)
+	obj.SetAttributes(append(obj.Attributes(), *attr)...)
+
+	r := new(fasthttp.RequestCtx)
+	utils.SetContextToRequest(ctx, r)
+
+	cid := cnrID.EncodeToString()
+	cid, err = maybeFillRandom(tp, cid)
+	if err != nil {
+		return nil, err
+	}
+	oid := putRes.ObjectID
+	oid, err = maybeFillRandom(tp, oid)
+	if err != nil {
+		return nil, err
+	}
+	r.SetUserValue("cid", cid)
+	r.SetUserValue("oid", oid)
+
+	rnd, err := tp.GetBool()
+	if err != nil {
+		return nil, err
+	}
+	if rnd == true {
+		r.SetUserValue("download", "true")
+	}
+
+	return r, nil
+}
+
+func InitFuzzGet() {
+}
+
+func DoFuzzGet(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := go_fuzz_utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	ctx, hc, cnrID, resp, filename, _, _, err := upload(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	hc.Handler().DownloadByAddressOrBucketName(r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzGet(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzUpload(data)
+	})
+}
+
+func InitFuzzHead() {
+}
+
+func DoFuzzHead(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := go_fuzz_utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	ctx, hc, cnrID, resp, filename, _, _, err := upload(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	hc.Handler().HeadByAddressOrBucketName(r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzHead(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzHead(data)
+	})
+}
+
+func InitFuzzDownloadByAttribute() {
+}
+
+func DoFuzzDownloadByAttribute(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := go_fuzz_utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	cid := cnrID.EncodeToString()
+	cid, err = maybeFillRandom(tp, cid)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	attrKey, err = maybeFillRandom(tp, attrKey)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	attrVal, err = maybeFillRandom(tp, attrVal)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	r := new(fasthttp.RequestCtx)
+	utils.SetContextToRequest(ctx, r)
+	r.SetUserValue("cid", cid)
+	r.SetUserValue("attr_key", attrKey)
+	r.SetUserValue("attr_val", attrVal)
+
+	hc.Handler().DownloadByAttribute(r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzDownloadByAttribute(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzDownloadByAttribute(data)
+	})
+}
+
+func InitFuzzHeadByAttribute() {
+}
+
+func DoFuzzHeadByAttribute(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := go_fuzz_utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	cid := cnrID.EncodeToString()
+	cid, err = maybeFillRandom(tp, cid)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	attrKey, err = maybeFillRandom(tp, attrKey)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	attrVal, err = maybeFillRandom(tp, attrVal)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	r := new(fasthttp.RequestCtx)
+	utils.SetContextToRequest(ctx, r)
+	r.SetUserValue("cid", cid)
+	r.SetUserValue("attr_key", attrKey)
+	r.SetUserValue("attr_val", attrVal)
+
+	hc.Handler().HeadByAttribute(r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzHeadByAttribute(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzHeadByAttribute(data)
+	})
+}
+
+func InitFuzzDownloadZipped() {
+}
+
+func DoFuzzDownloadZipped(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := go_fuzz_utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	ctx, hc, cnrID, _, _, _, _, err := upload(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	cid := cnrID.EncodeToString()
+	cid, err = maybeFillRandom(tp, cid)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	prefix := ""
+	prefix, err = maybeFillRandom(tp, prefix)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	r := new(fasthttp.RequestCtx)
+	utils.SetContextToRequest(ctx, r)
+	r.SetUserValue("cid", cid)
+	r.SetUserValue("prefix", prefix)
+
+	hc.Handler().DownloadZipped(r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzDownloadZipped(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzDownloadZipped(data)
+	})
+}
+
+func InitFuzzStoreBearerTokenAppCtx() {
+}
+
+func DoFuzzStoreBearerTokenAppCtx(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := go_fuzz_utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	prefix := ""
+	prefix, err = maybeFillRandom(tp, prefix)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	ctx := context.Background()
+	ctx = middleware.SetNamespace(ctx, "")
+
+	r := new(fasthttp.RequestCtx)
+	utils.SetContextToRequest(ctx, r)
+
+	strings, err := prepareStrings(tp, 3)
+
+	rand, err := prepareBools(tp, 2)
+
+	if rand[0] == true {
+		r.Request.Header.Set(fasthttp.HeaderAuthorization, "Bearer"+strings[0])
+	} else if rand[1] == true {
+		r.Request.Header.SetCookie(fasthttp.HeaderAuthorization, "Bearer"+strings[1])
+	} else {
+		r.Request.Header.Set(fasthttp.HeaderAuthorization, "Bearer"+strings[0])
+		r.Request.Header.SetCookie(fasthttp.HeaderAuthorization, "Bearer"+strings[1])
+	}
+
+	tokens.StoreBearerTokenAppCtx(ctx, r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzStoreBearerTokenAppCtx(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzStoreBearerTokenAppCtx(data)
+	})
+}