From e5eb732cd51409b21f549ef8cfcbc70dd10deb39 Mon Sep 17 00:00:00 2001 From: Roman Ognev Date: Mon, 17 Jun 2024 14:36:53 +0300 Subject: [PATCH] [#480] Add fuzzing tests (cherry picked from commit 664f83b2b744eac8fc8ccbd0b4c4d67a642ce1e3) Signed-off-by: Roman Ognev Signed-off-by: Alex Vanin --- Makefile | 34 ++ README.md | 18 + api/auth/center_fuzz_test.go | 88 +++ api/handler/handler_fuzz_test.go | 998 +++++++++++++++++++++++++++++++ api/handler/handlers_test.go | 54 +- api/handler/head_test.go | 12 +- api/handler/object_list_test.go | 7 +- go.mod | 1 + go.sum | 2 + 9 files changed, 1194 insertions(+), 20 deletions(-) create mode 100644 api/auth/center_fuzz_test.go create mode 100644 api/handler/handler_fuzz_test.go diff --git a/Makefile b/Makefile index e074da2..9a6ee92 100755 --- a/Makefile +++ b/Makefile @@ -23,6 +23,12 @@ OUTPUT_LINT_DIR ?= $(shell pwd)/bin LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION) TMP_DIR := .cache +# Variables for fuzzing +FUZZ_NGFUZZ_DIR ?= "" +FUZZ_TIMEOUT ?= 30 +FUZZ_FUNCTIONS ?= "all" +FUZZ_AUX ?= "" + .PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc # .deb package versioning @@ -76,6 +82,34 @@ cover: @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic @go tool cover -html=coverage.txt -o coverage.html +# Run fuzzing +CLANG := $(shell which clang-17 2>/dev/null) +.PHONY: check-clang all +check-clang: +ifeq ($(CLANG),) + @echo "clang-17 is not installed. Please install it before proceeding - https://apt.llvm.org/llvm.sh " + @exit 1 +endif + +.PHONY: check-ngfuzz all +check-ngfuzz: + @if [ -z "$(FUZZ_NGFUZZ_DIR)" ]; then \ + echo "Please set a variable FUZZ_NGFUZZ_DIR to specify path to the ngfuzz"; \ + exit 1; \ + fi + +.PHONY: install-fuzzing-deps +install-fuzzing-deps: check-clang check-ngfuzz + +.PHONY: fuzz +fuzz: install-fuzzing-deps + @START_PATH=$$(pwd); \ + ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \ + cd $(FUZZ_NGFUZZ_DIR) && \ + ./ngfuzz -clean && \ + ./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \ + ./ngfuzz -report + # Reformat code format: @echo "⇒ Processing gofmt check" diff --git a/README.md b/README.md index bf76331..26194fb 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,24 @@ HTTP/1.1 200 OK Also, you can configure domains using `.env` variables or `yaml` file. +## Fuzzing +To run fuzzing tests use the following command: + +```shell +$ make fuzz +``` + +This command will install dependencies for the fuzzing process and run existing fuzzing tests. + +You can also use the following arguments: + +``` +FUZZ_TIMEOUT - time to run each fuzzing test (default 30) +FUZZ_FUNCTIONS - fuzzing tests that will be started (default "all") +FUZZ_AUX - additional parameters for the fuzzer (for example, "-debug") +FUZZ_NGFUZZ_DIR - path to ngfuzz tool +```` + ## Documentation - [Configuration](./docs/configuration.md) diff --git a/api/auth/center_fuzz_test.go b/api/auth/center_fuzz_test.go new file mode 100644 index 0000000..950e54d --- /dev/null +++ b/api/auth/center_fuzz_test.go @@ -0,0 +1,88 @@ +//go:build gofuzz +// +build gofuzz + +package auth + +import ( + "strings" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/aws/aws-sdk-go/aws/credentials" + utils "github.com/trailofbits/go-fuzz-utils" +) + +const ( + fuzzSuccessExitCode = 0 + fuzzFailExitCode = -1 +) + +func InitFuzzAuthenticate() { +} + +func DoFuzzAuthenticate(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + var accessKeyAddr oid.Address + err = tp.Fill(accessKeyAddr) + if err != nil { + return fuzzFailExitCode + } + + accessKeyID := strings.ReplaceAll(accessKeyAddr.String(), "/", "0") + secretKey, err := tp.GetString() + awsCreds := credentials.NewStaticCredentials(accessKeyID, secretKey, "") + + reqData := RequestData{ + Method: "GET", + Endpoint: "http://localhost:8084", + Bucket: "my-bucket", + Object: "@obj/name", + } + presignData := PresignData{ + Service: "s3", + Region: "spb", + Lifetime: 10 * time.Minute, + SignTime: time.Now().UTC(), + } + + req, err := PresignRequest(awsCreds, reqData, presignData) + if req == nil { + return fuzzFailExitCode + } + + expBox := &accessbox.Box{ + Gate: &accessbox.GateData{ + SecretKey: secretKey, + }, + } + + mock := newTokensFrostfsMock() + mock.addBox(accessKeyAddr, expBox) + + c := &Center{ + cli: mock, + reg: NewRegexpMatcher(authorizationFieldRegexp), + postReg: NewRegexpMatcher(postPolicyCredentialRegexp), + } + + _, _ = c.Authenticate(req) + + return fuzzSuccessExitCode +} + +func FuzzAuthenticate(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzAuthenticate(data) + }) +} diff --git a/api/handler/handler_fuzz_test.go b/api/handler/handler_fuzz_test.go new file mode 100644 index 0000000..0499930 --- /dev/null +++ b/api/handler/handler_fuzz_test.go @@ -0,0 +1,998 @@ +//go:build gofuzz +// +build gofuzz + +package handler + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + tt "testing" // read https://github.com/AdamKorcz/go-118-fuzz-build?tab=readme-ov-file#workflow + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" + engineiam "git.frostfs.info/TrueCloudLab/policy-engine/iam" + utils "github.com/trailofbits/go-fuzz-utils" + "go.uber.org/zap/zaptest" +) + +var ( + fuzzBktName string + fuzzBox *accessbox.Box + fuzzHc *handlerContextBase + fuzzt *tt.T +) + +const ( + fuzzSuccessExitCode = 0 + fuzzFailExitCode = -1 +) + +func createTestBucketAndInitContext() { + fuzzt = new(tt.T) + + log := zaptest.NewLogger(fuzzt) + var err error + fuzzHc, err = prepareHandlerContextBase(layer.DefaultCachesConfigs(log)) + if err != nil { + panic(err) + } + + fuzzBktName = "bucket" + fuzzBox, _ = createAccessBox(fuzzt) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, defaultURL, nil) + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + fuzzHc.Handler().CreateBucketHandler(w, r) +} + +func prepareStrings(tp *utils.TypeProvider, count int) ([]string, error) { + array := make([]string, count) + var err error + + for i := 0; i < count; i++ { + err = tp.Reset() + if err != nil { + return nil, err + } + + array[i], err = tp.GetString() + if err != nil { + return nil, err + } + } + + return array, nil +} + +func addMD5Header(tp *utils.TypeProvider, r *http.Request, rawBody []byte) error { + if len(rawBody) == 0 { + return nil + } + + rand, err := tp.GetBool() + if err != nil { + return err + } + + if rand == true { + var dst []byte + base64.StdEncoding.Encode(dst, rawBody) + hash := md5.Sum(dst) + r.Header.Set("Content-Md5", hex.EncodeToString(hash[:])) + } + + return nil +} + +func generateParams(tp *utils.TypeProvider, input string, params []string) (string, error) { + input += "?" + + count, err := tp.GetInt() + if err != nil { + return "", err + } + count = count % len(params) + if count < 0 { + count += len(params) + } + + for i := 0; i < count; i++ { + position, err := tp.GetInt() + if err != nil { + return "", err + } + position = position % len(params) + if position < 0 { + position += len(params) + } + + v, err := tp.GetString() + if err != nil { + return "", err + } + input += params[position] + "=" + v + "&" + } + + return input, nil +} + +func generateHeaders(tp *utils.TypeProvider, r *http.Request, params []string) error { + count, err := tp.GetInt() + if err != nil { + return err + } + count = count % len(params) + if count < 0 { + count += len(params) + } + + for i := 0; i < count; i++ { + position, err := tp.GetInt() + if err != nil { + return err + } + position = position % len(params) + if position < 0 { + position += len(params) + } + + v, err := tp.GetString() + if err != nil { + return err + } + + r.Header.Set(params[position], v) + + } + + return nil +} + +func InitFuzzCreateBucketHandler() { + fuzzt = new(tt.T) + + log := zaptest.NewLogger(fuzzt) + var err error + fuzzHc, err = prepareHandlerContextBase(layer.DefaultCachesConfigs(log)) + if err != nil { + panic(err) + } + + fuzzBox, _ = createAccessBox(fuzzt) +} + +func DoFuzzCreateBucketHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + strings, err := prepareStrings(tp, 4) + if err != nil { + return fuzzFailExitCode + } + + bktName := strings[0] + body := strings[1] + + bodyXml, err := xml.Marshal(body) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(bodyXml)) + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-acl", "x-amz-bucket-object-lock-enabled", "x-amz-grant-full-control", "x-amz-grant-read", "x-amz-grant-read-acp", "x-amz-grant-write", "x-amz-grant-write-acp", "x-amz-object-ownership"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().CreateBucketHandler(w, r) + + return fuzzSuccessExitCode +} + +func FuzzCreateBucketHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzCreateBucketHandler(data) + }) +} + +func InitFuzzPutBucketCorsHandler() { + createTestBucketAndInitContext() +} + +func DoFuzzPutBucketCorsHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + var cors data.CORSConfiguration + err = tp.Fill(&cors) + if err != nil { + return fuzzFailExitCode + } + + bodyXml, err := xml.Marshal(cors) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, defaultURL+"?cors", bytes.NewReader(bodyXml)) + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().PutBucketCorsHandler(w, r) + + return fuzzSuccessExitCode +} + +func FuzzPutBucketCorsHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzPutBucketCorsHandler(data) + }) +} + +func InitFuzzPutBucketPolicyHandler() { + createTestBucketAndInitContext() +} + +func FuzzPutBucketPolicyHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzPutBucketPolicyHandler(data) + }) +} + +func DoFuzzPutBucketPolicyHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + var policy engineiam.Policy + err = tp.Fill(&policy) + if err != nil { + return fuzzFailExitCode + } + + bodyXml, err := xml.Marshal(policy) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, defaultURL+"?policy", bytes.NewReader(bodyXml)) + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-confirm-remove-self-bucket-access"}) + if err != nil { + return fuzzFailExitCode + } + + err = addMD5Header(tp, r, bodyXml) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().PutBucketPolicyHandler(w, r) + + return fuzzSuccessExitCode +} + +func InitFuzzDeleteMultipleObjectsHandler() { + createTestBucketAndInitContext() +} + +func FuzzDeleteMultipleObjectsHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDeleteMultipleObjectsHandler(data) + }) +} + +func DoFuzzDeleteMultipleObjectsHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + var body DeleteObjectsRequest + err = tp.Fill(&body) + if err != nil { + return fuzzFailExitCode + } + + bodyXml, err := xml.Marshal(body) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, defaultURL+"?delete", bytes.NewReader(bodyXml)) + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bypass-governance-retention", "x-amz-mfa"}) + if err != nil { + return fuzzFailExitCode + } + + err = addMD5Header(tp, r, bodyXml) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().DeleteMultipleObjectsHandler(w, r) + + return fuzzSuccessExitCode +} + +func InitFuzzPostObject() { + createTestBucketAndInitContext() +} + +func FuzzPostObject(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzPostObject(data) + }) +} + +func postObject(tp *utils.TypeProvider) ([]byte, string, error) { + strings, err := prepareStrings(tp, 2) + if err != nil { + return nil, "", err + } + + bodyXml, err := xml.Marshal(strings[0]) + if err != nil { + return nil, "", err + } + objName := strings[1] + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, defaultURL, bytes.NewReader(bodyXml)) + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"X-Amz-Grant-Read", "X-Amz-Grant-Full-Control", "X-Amz-Grant-Write", "X-Amz-Acl", "x-amz-expected-bucket-owner"}) + if err != nil { + return nil, "", err + } + + var file multipart.Form + err = tp.Fill(&file) + if err != nil { + return nil, "", err + } + + r.MultipartForm = &file + + fuzzHc.Handler().PostObject(w, r) + + return bodyXml, objName, nil +} + +func DoFuzzPostObject(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + _, _, err = postObject(tp) + if err != nil { + return fuzzFailExitCode + } + + return fuzzSuccessExitCode +} + +func InitFuzzDeleteBucketHandler() { + createTestBucketAndInitContext() +} + +func FuzzDeleteBucketHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDeleteBucketHandler(data) + }) +} + +func DoFuzzDeleteBucketHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodDelete, defaultURL, nil) + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().DeleteBucketHandler(w, r) + + return fuzzSuccessExitCode +} + +func InitFuzzDeleteBucketCorsHandler() { + createTestBucketAndInitContext() +} + +func FuzzDeleteBucketCorsHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDeleteBucketCorsHandler(data) + }) +} + +func DoFuzzDeleteBucketCorsHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodDelete, defaultURL+"?cors", nil) + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().DeleteBucketCorsHandler(w, r) + + return fuzzSuccessExitCode +} + +func InitFuzzDeleteBucketPolicyHandler() { + createTestBucketAndInitContext() +} + +func FuzzDeleteBucketPolicyHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDeleteBucketPolicyHandler(data) + }) +} + +func DoFuzzDeleteBucketPolicyHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodDelete, defaultURL+"?policy", nil) + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().DeleteBucketPolicyHandler(w, r) + + return fuzzFailExitCode +} + +func InitFuzzCopyObjectHandler() { + createTestBucketAndInitContext() +} + +func FuzzCopyObjectHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzCopyObjectHandler(data) + }) +} + +func DoFuzzCopyObjectHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + var r *http.Request + + key, err := tp.GetString() + if err != nil { + return fuzzFailExitCode + } + + params, err := generateParams(tp, key, []string{"versionId"}) + if err != nil { + return fuzzFailExitCode + } + + r = httptest.NewRequest(http.MethodPut, defaultURL+params, nil) + if r != nil { + return fuzzFailExitCode + } + + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-acl", "x-amz-checksum-algorithm", "x-amz-copy-source", "x-amz-copy-source-if-match", "x-amz-copy-source-if-match", "x-amz-copy-source-if-unmodified-since", "x-amz-copy-source-if-modified-since", "x-amz-copy-source-if-none-match", "x-amz-copy-source-if-modified-since", "x-amz-copy-source-if-none-match", "x-amz-copy-source-if-none-match", "x-amz-copy-source-if-modified-since", "x-amz-copy-source-if-unmodified-since", "x-amz-copy-source-if-match", "x-amz-copy-source-if-unmodified-since", "x-amz-copy-source-server-side-encryption-customer-algorithm", "x-amz-copy-source-server-side-encryption-customer-key", "x-amz-copy-source-server-side-encryption-customer-key-MD5", "x-amz-expected-bucket-owner", "x-amz-grant-full-control", "x-amz-grant-read", "x-amz-grant-read-acp", "x-amz-grant-write-acp", "x-amz-metadata-directive", "x-amz-website-redirect-location", "x-amz-object-lock-legal-hold", "x-amz-object-lock-mode", "x-amz-object-lock-retain-until-date", "x-amz-request-payer", "x-amz-server-side-encryption", "x-amz-server-side-encryption-aws-kms-key-id", "x-amz-server-side-encryption-bucket-key-enabled", "x-amz-server-side-encryption-context", "x-amz-server-side-encryption-customer-algorithm", "x-amz-server-side-encryption-customer-key", "x-amz-server-side-encryption-customer-key-MD5", "x-amz-source-expected-bucket-owner", "x-amz-storage-class", "x-amz-tagging", "x-amz-tagging-directive", "x-amz-website-redirect-location"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().CopyObjectHandler(w, r) + + return fuzzSuccessExitCode +} + +func InitFuzzDeleteObjectHandler() { + createTestBucketAndInitContext() +} + +func FuzzDeleteObjectHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDeleteObjectHandler(data) + }) +} + +func DoFuzzDeleteObjectHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + _, objName, err := postObject(tp) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + var r *http.Request + + params, err := generateParams(tp, objName, []string{"versionId"}) + if err != nil { + return fuzzFailExitCode + } + + r = httptest.NewRequest(http.MethodDelete, defaultURL+params, nil) + if r != nil { + return fuzzFailExitCode + } + + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bypass-governance-retention", "x-amz-mfa"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().DeleteObjectHandler(w, r) + + return fuzzSuccessExitCode +} + +func InitFuzzGetObjectHandler() { + createTestBucketAndInitContext() +} + +func FuzzGetObjectHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzGetObjectHandler(data) + }) +} + +func DoFuzzGetObjectHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + _, objName, err := postObject(tp) + if err != nil { + return fuzzFailExitCode + } + + params, err := generateParams(tp, objName, []string{"versionId", "partNumber", "Range", "response-content-type", "response-content-language", "response-expires", "response-cache-control", "response-content-disposition", "response-content-encoding"}) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + + r := httptest.NewRequest(http.MethodGet, defaultURL+params, nil) + if r != nil { + return fuzzFailExitCode + } + + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "If-Match", "If-None-Match", "If-Modified-Since", "If-Unmodified-Since", "x-amz-server-side-encryption-customer-algorithm", "x-amz-server-side-encryption-customer-key", "x-amz-server-side-encryption-customer-key-MD5", "Range"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().GetObjectHandler(w, r) + + return fuzzSuccessExitCode +} + +func InitFuzzPutObjectHandler() { + createTestBucketAndInitContext() +} + +func DoFuzzPutObjectHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + objName, err := tp.GetString() + if err != nil { + return fuzzFailExitCode + } + + body, err := tp.GetBytes() + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + + r := httptest.NewRequest(http.MethodPut, defaultURL+objName, bytes.NewReader(body)) + if r != nil { + return fuzzFailExitCode + } + + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "X-Amz-Grant-Read", "X-Amz-Grant-Full-Control", "X-Amz-Grant-Write", "X-Amz-Acl", "X-Amz-Tagging", "Content-Type", "Cache-Control", "Expires", "Content-Language", "Content-Encoding", "x-amz-server-side-encryption-customer-algorithm", "x-amz-server-side-encryption-customer-key", "x-amz-server-side-encryption-customer-key-MD5", "X-Amz-Content-Sha256", "X-Amz-Object-Lock-Legal-Hold", "X-Amz-Object-Lock-Mode", "X-Amz-Object-Lock-Retain-Until-Date", "X-Amz-Bypass-Governance-Retention", "X-Amz-Meta-*"}) + if err != nil { + return fuzzFailExitCode + } + + err = addMD5Header(tp, r, body) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().PutObjectHandler(w, r) + + return fuzzSuccessExitCode +} + +func FuzzPutObjectHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzPutObjectHandler(data) + }) +} + +func InitFuzzPutObjectLegalHoldHandler() { + createTestBucketAndInitContext() +} + +func DoFuzzPutObjectLegalHoldHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + _, objName, err := postObject(tp) + if err != nil { + return fuzzFailExitCode + } + + var hold data.LegalHold + err = tp.Fill(&hold) + if err != nil { + return fuzzFailExitCode + } + + rawBody, err := xml.Marshal(hold) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + + r := httptest.NewRequest(http.MethodPut, defaultURL+objName+"?legal-hold", bytes.NewReader(rawBody)) + if r != nil { + return fuzzFailExitCode + } + + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = addMD5Header(tp, r, rawBody) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().PutObjectLegalHoldHandler(w, r) + + return fuzzSuccessExitCode +} + +func FuzzPutObjectLegalHoldHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzPutObjectLegalHoldHandler(data) + }) +} + +func InitFuzzPutBucketObjectLockConfigHandler() { + createTestBucketAndInitContext() +} + +func DoFuzzPutBucketObjectLockConfigHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + var hold data.ObjectLockConfiguration + err = tp.Fill(&hold) + if err != nil { + return fuzzFailExitCode + } + + rawBody, err := xml.Marshal(&hold) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + + r := httptest.NewRequest(http.MethodPut, defaultURL+"?object-lock", bytes.NewReader(rawBody)) + if r != nil { + return fuzzFailExitCode + } + + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = addMD5Header(tp, r, rawBody) + if err != nil { + return fuzzFailExitCode + } + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bucket-object-lock-token"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().PutBucketObjectLockConfigHandler(w, r) + + return fuzzSuccessExitCode +} + +func FuzzPutBucketObjectLockConfigHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzPutBucketObjectLockConfigHandler(data) + }) +} + +func InitFuzzPutObjectRetentionHandler() { + createTestBucketAndInitContext() +} + +func DoFuzzPutObjectRetentionHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + _, objName, err := postObject(tp) + if err != nil { + return fuzzFailExitCode + } + + var retention data.Retention + err = tp.Fill(&retention) + if err != nil { + return fuzzFailExitCode + } + + rawBody, err := xml.Marshal(retention) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + + r := httptest.NewRequest(http.MethodPut, defaultURL+objName+"?retention", bytes.NewReader(rawBody)) + if r != nil { + return fuzzFailExitCode + } + + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = addMD5Header(tp, r, rawBody) + if err != nil { + return fuzzFailExitCode + } + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bypass-governance-retention"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().PutObjectRetentionHandler(w, r) + + return fuzzSuccessExitCode +} + +func FuzzPutObjectRetentionHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzPutObjectRetentionHandler(data) + }) +} + +func InitFuzzPutBucketAclHandler() { + createTestBucketAndInitContext() +} + +func DoFuzzPutBucketAclHandler(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + var policy AccessControlPolicy + err = tp.Fill(&policy) + if err != nil { + return fuzzFailExitCode + } + + rawBody, err := xml.Marshal(policy) + if err != nil { + return fuzzFailExitCode + } + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, defaultURL+"?acl", bytes.NewReader(rawBody)) + reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "") + r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo)) + r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox})) + + err = addMD5Header(tp, r, rawBody) + if err != nil { + return fuzzFailExitCode + } + + err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-acl", "x-amz-expected-bucket-owner", "x-amz-grant-full-control", "x-amz-grant-read", "x-amz-grant-read-acp", "x-amz-grant-write", "x-amz-grant-write-acp"}) + if err != nil { + return fuzzFailExitCode + } + + fuzzHc.Handler().PutBucketACLHandler(w, r) + + return fuzzSuccessExitCode +} + +func FuzzPutBucketAclHandler(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzPutBucketAclHandler(data) + }) +} diff --git a/api/handler/handlers_test.go b/api/handler/handlers_test.go index 254087a..eb06207 100644 --- a/api/handler/handlers_test.go +++ b/api/handler/handlers_test.go @@ -37,8 +37,12 @@ import ( ) type handlerContext struct { + *handlerContextBase + t *testing.T +} + +type handlerContextBase struct { owner user.ID - t *testing.T h *handler tp *layer.TestFrostFS tree *tree.Tree @@ -50,19 +54,19 @@ type handlerContext struct { cache *layer.Cache } -func (hc *handlerContext) Handler() *handler { +func (hc *handlerContextBase) Handler() *handler { return hc.h } -func (hc *handlerContext) MockedPool() *layer.TestFrostFS { +func (hc *handlerContextBase) MockedPool() *layer.TestFrostFS { return hc.tp } -func (hc *handlerContext) Layer() *layer.Layer { +func (hc *handlerContextBase) Layer() *layer.Layer { return hc.h.obj } -func (hc *handlerContext) Context() context.Context { +func (hc *handlerContextBase) Context() context.Context { return hc.context } @@ -136,16 +140,28 @@ func (c *configMock) RetryStrategy() RetryStrategy { } func prepareHandlerContext(t *testing.T) *handlerContext { - return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(zap.NewExample())) + hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(zap.NewExample())) + require.NoError(t, err) + return &handlerContext{ + handlerContextBase: hc, + t: t, + } } func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext { - return prepareHandlerContextBase(t, getMinCacheConfig(zap.NewExample())) + hc, err := prepareHandlerContextBase(getMinCacheConfig(zap.NewExample())) + require.NoError(t, err) + return &handlerContext{ + handlerContextBase: hc, + t: t, + } } -func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *handlerContext { +func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBase, error) { key, err := keys.NewPrivateKey() - require.NoError(t, err) + if err != nil { + return nil, err + } l := zap.NewExample() tp := layer.NewTestFrostFS(key) @@ -159,7 +175,9 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand user.IDFromKey(&owner, key.PrivateKey.PublicKey) memCli, err := tree.NewTreeServiceClientMemory() - require.NoError(t, err) + if err != nil { + return nil, err + } treeMock := tree.NewTree(memCli, zap.NewExample()) @@ -176,7 +194,9 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand var pp netmap.PlacementPolicy err = pp.DecodeString("REP 1") - require.NoError(t, err) + if err != nil { + return nil, err + } cfg := &configMock{ defaultPolicy: pp, @@ -189,19 +209,23 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand frostfsid: newFrostfsIDMock(), } - return &handlerContext{ + accessBox, err := newTestAccessBox(key) + if err != nil { + return nil, err + } + + return &handlerContextBase{ owner: owner, - t: t, h: h, tp: tp, tree: treeMock, - context: middleware.SetBox(context.Background(), &middleware.Box{AccessBox: newTestAccessBox(t, key)}), + context: middleware.SetBox(context.Background(), &middleware.Box{AccessBox: accessBox}), config: cfg, layerFeatures: features, treeMock: memCli, cache: layerCfg.Cache, - } + }, nil } func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig { diff --git a/api/handler/head_test.go b/api/handler/head_test.go index 2225c96..9f0c88f 100644 --- a/api/handler/head_test.go +++ b/api/handler/head_test.go @@ -119,21 +119,25 @@ func TestIsAvailableToResolve(t *testing.T) { } } -func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box { +func newTestAccessBox(key *keys.PrivateKey) (*accessbox.Box, error) { var err error if key == nil { key, err = keys.NewPrivateKey() - require.NoError(t, err) + if err != nil { + return nil, err + } } var btoken bearer.Token btoken.SetImpersonate(true) err = btoken.Sign(key.PrivateKey) - require.NoError(t, err) + if err != nil { + return nil, err + } return &accessbox.Box{ Gate: &accessbox.GateData{ BearerToken: &btoken, }, - } + }, nil } diff --git a/api/handler/object_list_test.go b/api/handler/object_list_test.go index 575fd53..f20abbb 100644 --- a/api/handler/object_list_test.go +++ b/api/handler/object_list_test.go @@ -142,7 +142,12 @@ func TestListObjectsContextCanceled(t *testing.T) { layerCfg.SessionList.Lifetime = time.Hour layerCfg.SessionList.Size = 1 - hc := prepareHandlerContextBase(t, layerCfg) + hcBase, err := prepareHandlerContextBase(layerCfg) + require.NoError(t, err) + hc := &handlerContext{ + handlerContextBase: hcBase, + t: t, + } bktName := "bucket-versioning-enabled" bktInfo := createTestBucket(hc, bktName) diff --git a/go.mod b/go.mod index 0f80440..0563d2e 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/spf13/viper v1.15.0 github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 + github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/urfave/cli/v2 v2.3.0 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 diff --git a/go.sum b/go.sum index ff6d3de..96c5562 100644 --- a/go.sum +++ b/go.sum @@ -321,6 +321,8 @@ github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 h1:GpfJ7OdNjS7BFTVwNCUI9L4aCJOFRbr5fdHqjdhoYE8= +github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4/go.mod h1:f3jBhpWvuZmue0HZK52GzRHJOYHYSILs/c8+K2S/J+o= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=