From 947f81fcdd995dcb829f670d76f960a842a8a943 Mon Sep 17 00:00:00 2001 From: Roman Ognev Date: Fri, 30 Aug 2024 20:06:01 +0300 Subject: [PATCH] [#135] Add fuzzing tests for handlers (cherry picked from commit ca426fff4df117b5ce119d7a5f123b11cc99a7a1) Signed-off-by: Roman Ognev --- Makefile | 36 +- README.md | 23 + go.mod | 1 + go.sum | 2 + internal/handler/handler_fuzz_test.go | 580 ++++++++++++++++++++++++++ 5 files changed, 641 insertions(+), 1 deletion(-) create mode 100644 internal/handler/handler_fuzz_test.go diff --git a/Makefile b/Makefile index 04cfea4..0d956a6 100755 --- a/Makefile +++ b/Makefile @@ -30,6 +30,11 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed "s/-/~/")-${OS_RELEASE} .PHONY: debpackage debclean +FUZZ_NGFUZZ_DIR ?= "" +FUZZ_TIMEOUT ?= 30 +FUZZ_FUNCTIONS ?= "all" +FUZZ_AUX ?= "" + # Make all binaries all: $(BINS) $(BINS): $(DIRS) dep @@ -78,6 +83,35 @@ cover: @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic @go tool cover -html=coverage.txt -o coverage.html +# Run fuzzing +CLANG := $(shell which clang-17 2>/dev/null) +.PHONY: check-clang all +check-clang: +ifeq ($(CLANG),) + @echo "clang-17 is not installed. Please install it before proceeding - https://apt.llvm.org/llvm.sh " + @exit 1 +endif + +.PHONY: check-ngfuzz all +check-ngfuzz: + @if [ -z "$(FUZZ_NGFUZZ_DIR)" ]; then \ + echo "Please set a variable FUZZ_NGFUZZ_DIR to specify path to the ngfuzz"; \ + exit 1; \ + fi + +.PHONY: install-fuzzing-deps +install-fuzzing-deps: check-clang check-ngfuzz + +.PHONY: fuzz +fuzz: install-fuzzing-deps + @START_PATH=$$(pwd); \ + ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \ + cd $(FUZZ_NGFUZZ_DIR) && \ + ./ngfuzz -clean && \ + ./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \ + ./ngfuzz -report + + # Reformat code fmt: @echo "⇒ Processing gofmt check" @@ -149,7 +183,7 @@ version: # Clean up clean: rm -rf vendor - rm -rf $(BINDIR) + rm -rf $(BINDIR) # Package for Debian debpackage: diff --git a/README.md b/README.md index aa982db..019b8ff 100644 --- a/README.md +++ b/README.md @@ -575,3 +575,26 @@ See [configuration](./docs/gate-configuration.md). ## Credits Please see [CREDITS](CREDITS.md) for details. + +## Fuzzing + +To run fuzzing tests use the following command: + +```shell +$ make fuzz +``` + +This command will install dependencies for the fuzzing process and run existing fuzzing tests. + +You can also use the following arguments: + +``` +FUZZ_TIMEOUT - time to run each fuzzing test (default 30) +FUZZ_FUNCTIONS - fuzzing tests that will be started (default "all") +FUZZ_AUX - additional parameters for the fuzzer (for example, "-debug") +FUZZ_NGFUZZ_DIR - path to ngfuzz tool +```` + +## Credits + +Please see [CREDITS](CREDITS.md) for details. diff --git a/go.mod b/go.mod index 5d9b8a1..768019b 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.13.0 + github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/valyala/fasthttp v1.34.0 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 diff --git a/go.sum b/go.sum index 35c857d..7f0419f 100644 --- a/go.sum +++ b/go.sum @@ -820,6 +820,8 @@ github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBN github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 h1:GpfJ7OdNjS7BFTVwNCUI9L4aCJOFRbr5fdHqjdhoYE8= +github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4/go.mod h1:f3jBhpWvuZmue0HZK52GzRHJOYHYSILs/c8+K2S/J+o= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= diff --git a/internal/handler/handler_fuzz_test.go b/internal/handler/handler_fuzz_test.go new file mode 100644 index 0000000..ad2ae6e --- /dev/null +++ b/internal/handler/handler_fuzz_test.go @@ -0,0 +1,580 @@ +//go:build gofuzz +// +build gofuzz + +package handler + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "mime/multipart" + "net/http" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + go_fuzz_utils "github.com/trailofbits/go-fuzz-utils" + "github.com/valyala/fasthttp" +) + +const ( + fuzzSuccessExitCode = 0 + fuzzFailExitCode = -1 +) + +func prepareStrings(tp *go_fuzz_utils.TypeProvider, count int) ([]string, error) { + array := make([]string, count) + var err error + + for i := 0; i < count; i++ { + err = tp.Reset() + if err != nil { + return nil, err + } + + array[i], err = tp.GetString() + if err != nil { + return nil, err + } + } + + return array, nil +} + +func prepareBools(tp *go_fuzz_utils.TypeProvider, count int) ([]bool, error) { + array := make([]bool, count) + var err error + + for i := 0; i < count; i++ { + err = tp.Reset() + if err != nil { + return nil, err + } + + array[i], err = tp.GetBool() + if err != nil { + return nil, err + } + } + + return array, nil +} + +func getRandomDeterministicPositiveIntInRange(tp *go_fuzz_utils.TypeProvider, max int) (int, error) { + count, err := tp.GetInt() + if err != nil { + return -1, err + } + count = count % max + if count < 0 { + count += max + } + return count, nil +} + +func generateHeaders(tp *go_fuzz_utils.TypeProvider, r *fasthttp.Request, params []string) error { + count, err := tp.GetInt() + if err != nil { + return err + } + count = count % len(params) + if count < 0 { + count += len(params) + } + + for i := 0; i < count; i++ { + position, err := tp.GetInt() + if err != nil { + return err + } + position = position % len(params) + if position < 0 { + position += len(params) + } + + v, err := tp.GetString() + if err != nil { + return err + } + + r.Header.Set(params[position], v) + + } + + return nil +} + +func maybeFillRandom(tp *go_fuzz_utils.TypeProvider, initValue string) (string, error) { + rnd, err := tp.GetBool() + if err != nil { + return "", err + } + if rnd == true { + initValue, err = tp.GetString() + if err != nil { + return "", err + } + } + return initValue, nil +} + +func upload(tp *go_fuzz_utils.TypeProvider) (context.Context, *handlerContext, cid.ID, *fasthttp.RequestCtx, string, string, string, error) { + hc, err := prepareHandlerContext() + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + aclList := []acl.Basic{ + acl.Private, + acl.PrivateExtended, + acl.PublicRO, + acl.PublicROExtended, + acl.PublicRW, + acl.PublicRWExtended, + acl.PublicAppend, + acl.PublicAppendExtended, + } + + pos, err := getRandomDeterministicPositiveIntInRange(tp, len(aclList)) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + acl := aclList[pos] + + strings, err := prepareStrings(tp, 6) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + bktName := strings[0] + objFileName := strings[1] + valAttr := strings[2] + keyAttr := strings[3] + + if len(bktName) == 0 { + return nil, nil, cid.ID{}, nil, "", "", "", errors.New("not enought buckets") + } + + cnrID, cnr, err := hc.prepareContainer(bktName, acl) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + hc.frostfs.SetContainer(cnrID, cnr) + + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cnrID.EncodeToString()) + + attributes := map[string]string{ + object.AttributeFileName: objFileName, + keyAttr: valAttr, + } + + var buff bytes.Buffer + w := multipart.NewWriter(&buff) + fw, err := w.CreateFormFile("file", attributes[object.AttributeFileName]) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + content, err := tp.GetBytes() + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + if _, err = io.Copy(fw, bytes.NewReader(content)); err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + if err = w.Close(); err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + r.Request.SetBodyStream(&buff, buff.Len()) + r.Request.Header.Set("Content-Type", w.FormDataContentType()) + r.Request.Header.Set("X-Attribute-"+keyAttr, valAttr) + + err = generateHeaders(tp, &r.Request, []string{"X-Attribute-", "X-Attribute-DupKey", "X-Attribute-MyAttribute", "X-Attribute-System-DupKey", "X-Attribute-System-Expiration-Epoch1", "X-Attribute-SYSTEM-Expiration-Epoch2", "X-Attribute-system-Expiration-Epoch3", "X-Attribute-User-Attribute", "X-Attribute-", "X-Attribute-FileName", "X-Attribute-FROSTFS", "X-Attribute-neofs", "X-Attribute-SYSTEM", "X-Attribute-System-Expiration-Duration", "X-Attribute-System-Expiration-Epoch", "X-Attribute-System-Expiration-RFC3339", "X-Attribute-System-Expiration-Timestamp", "X-Attribute-Timestamp", "X-Attribute-" + strings[4], "X-Attribute-System-" + strings[5]}) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + hc.Handler().Upload(r) + + if r.Response.StatusCode() != http.StatusOK { + return nil, nil, cid.ID{}, nil, "", "", "", errors.New("error on upload") + } + + return ctx, hc, cnrID, r, objFileName, keyAttr, valAttr, nil +} + +func InitFuzzUpload() { + +} + +func DoFuzzUpload(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + _, _, _, _, _, _, _, err = upload(tp) + if err != nil { + return fuzzFailExitCode + } + + return fuzzSuccessExitCode +} + +func FuzzUpload(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzUpload(data) + }) +} + +func downloadOrHead(tp *go_fuzz_utils.TypeProvider, ctx context.Context, hc *handlerContext, cnrID cid.ID, resp *fasthttp.RequestCtx, filename string) (*fasthttp.RequestCtx, error) { + + var putRes putResponse + + defer func() { + if r := recover(); r != nil { + panic(resp) + } + }() + + data := resp.Response.Body() + err := json.Unmarshal(data, &putRes) + + if err != nil { + return nil, err + } + + obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] + attr := object.NewAttribute() + attr.SetKey(object.AttributeFilePath) + + filename, err = maybeFillRandom(tp, filename) + if err != nil { + return nil, err + } + + attr.SetValue(filename) + obj.SetAttributes(append(obj.Attributes(), *attr)...) + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return nil, err + } + oid := putRes.ObjectID + oid, err = maybeFillRandom(tp, oid) + if err != nil { + return nil, err + } + r.SetUserValue("cid", cid) + r.SetUserValue("oid", oid) + + rnd, err := tp.GetBool() + if err != nil { + return nil, err + } + if rnd == true { + r.SetUserValue("download", "true") + } + + return r, nil +} + +func InitFuzzGet() { + +} + +func DoFuzzGet(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, resp, filename, _, _, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename) + if err != nil { + return fuzzFailExitCode + } + + hc.Handler().DownloadByAddressOrBucketName(r) + + return fuzzSuccessExitCode +} + +func FuzzGet(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzUpload(data) + }) +} + +func InitFuzzHead() { + +} + +func DoFuzzHead(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, resp, filename, _, _, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename) + if err != nil { + return fuzzFailExitCode + } + + hc.Handler().HeadByAddressOrBucketName(r) + + return fuzzSuccessExitCode +} + +func FuzzHead(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzHead(data) + }) +} + +func InitFuzzDownloadByAttribute() { + +} + +func DoFuzzDownloadByAttribute(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return fuzzFailExitCode + } + + attrKey, err = maybeFillRandom(tp, attrKey) + if err != nil { + return fuzzFailExitCode + } + + attrVal, err = maybeFillRandom(tp, attrVal) + if err != nil { + return fuzzFailExitCode + } + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cid) + r.SetUserValue("attr_key", attrKey) + r.SetUserValue("attr_val", attrVal) + + hc.Handler().DownloadByAttribute(r) + + return fuzzSuccessExitCode +} + +func FuzzDownloadByAttribute(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDownloadByAttribute(data) + }) +} + +func InitFuzzHeadByAttribute() { + +} + +func DoFuzzHeadByAttribute(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return fuzzFailExitCode + } + + attrKey, err = maybeFillRandom(tp, attrKey) + if err != nil { + return fuzzFailExitCode + } + + attrVal, err = maybeFillRandom(tp, attrVal) + if err != nil { + return fuzzFailExitCode + } + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cid) + r.SetUserValue("attr_key", attrKey) + r.SetUserValue("attr_val", attrVal) + + hc.Handler().HeadByAttribute(r) + + return fuzzSuccessExitCode +} + +func FuzzHeadByAttribute(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzHeadByAttribute(data) + }) +} + +func InitFuzzDownloadZipped() { + +} + +func DoFuzzDownloadZipped(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, _, _, _, _, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return fuzzFailExitCode + } + + prefix := "" + prefix, err = maybeFillRandom(tp, prefix) + if err != nil { + return fuzzFailExitCode + } + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cid) + r.SetUserValue("prefix", prefix) + + hc.Handler().DownloadZipped(r) + + return fuzzSuccessExitCode +} + +func FuzzDownloadZipped(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDownloadZipped(data) + }) +} + +func InitFuzzStoreBearerTokenAppCtx() { + +} + +func DoFuzzStoreBearerTokenAppCtx(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + prefix := "" + prefix, err = maybeFillRandom(tp, prefix) + if err != nil { + return fuzzFailExitCode + } + + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + + strings, err := prepareStrings(tp, 3) + + rand, err := prepareBools(tp, 2) + + if rand[0] == true { + r.Request.Header.Set(fasthttp.HeaderAuthorization, "Bearer"+strings[0]) + } else if rand[1] == true { + r.Request.Header.SetCookie(fasthttp.HeaderAuthorization, "Bearer"+strings[1]) + } else { + r.Request.Header.Set(fasthttp.HeaderAuthorization, "Bearer"+strings[0]) + r.Request.Header.SetCookie(fasthttp.HeaderAuthorization, "Bearer"+strings[1]) + } + + tokens.StoreBearerTokenAppCtx(ctx, r) + + return fuzzSuccessExitCode +} + +func FuzzStoreBearerTokenAppCtx(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzStoreBearerTokenAppCtx(data) + }) +}