frostfs-s3-gw/api/handler/patch_test.go

package handler

import (
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"encoding/xml"
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"strings"
	"testing"
	"time"

	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
	s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
	"github.com/stretchr/testify/require"
)

func TestPatch(t *testing.T) {
	tc := prepareHandlerContext(t)
	tc.config.md5Enabled = true

	bktName, objName := "bucket-for-patch", "object-for-patch"
	createTestBucket(tc, bktName)

	content := []byte("old object content")
	md5Hash := md5.New()
	md5Hash.Write(content)
	etag := data.Quote(hex.EncodeToString(md5Hash.Sum(nil)))

	w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
	created := time.Now()
	tc.Handler().PutObjectHandler(w, r)
	require.Equal(t, etag, w.Header().Get(api.ETag))

	patchPayload := []byte("new")
	sha256Hash := sha256.New()
	sha256Hash.Write(patchPayload)
	sha256Hash.Write(content[len(patchPayload):])
	hash := hex.EncodeToString(sha256Hash.Sum(nil))

	for _, tt := range []struct {
		name    string
		rng     string
		headers map[string]string
		code    s3errors.ErrorCode
	}{
		{
			name: "success",
			rng:  "bytes 0-2/*",
			headers: map[string]string{
				api.IfUnmodifiedSince: created.Format(http.TimeFormat),
				api.IfMatch:           etag,
			},
		},
		{
			name: "invalid range syntax",
			rng:  "bytes 0-2",
			code: s3errors.ErrInvalidRange,
		},
		{
			name: "invalid range length",
			rng:  "bytes 0-5/*",
			code: s3errors.ErrInvalidRangeLength,
		},
		{
			name: "invalid range start",
			rng:  "bytes 20-22/*",
			code: s3errors.ErrRangeOutOfBounds,
		},
		{
			name: "range is too long",
			rng:  "bytes 0-5368709120/*",
			code: s3errors.ErrInvalidRange,
		},
		{
			name: "If-Unmodified-Since precondition are not satisfied",
			rng:  "bytes 0-2/*",
			headers: map[string]string{
				api.IfUnmodifiedSince: created.Add(-24 * time.Hour).Format(http.TimeFormat),
			},
			code: s3errors.ErrPreconditionFailed,
		},
		{
			name: "If-Match precondition are not satisfied",
			rng:  "bytes 0-2/*",
			headers: map[string]string{
				api.IfMatch: "etag",
			},
			code: s3errors.ErrPreconditionFailed,
		},
	} {
		t.Run(tt.name, func(t *testing.T) {
			if tt.code == 0 {
				res := patchObject(t, tc, bktName, objName, tt.rng, patchPayload, tt.headers)
				require.Equal(t, data.Quote(hash), res.Object.ETag)
			} else {
				patchObjectErr(t, tc, bktName, objName, tt.rng, patchPayload, tt.headers, tt.code)
			}
		})
	}
}

func TestPatchMultipartObject(t *testing.T) {
	tc := prepareHandlerContextWithMinCache(t)
	tc.config.md5Enabled = true

	bktName, objName, partSize := "bucket-for-multipart-patch", "object-for-multipart-patch", 5*1024*1024
	createTestBucket(tc, bktName)

	t.Run("patch beginning of the first part", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
		etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
		etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})

		patchSize := partSize / 2
		patchBody := make([]byte, patchSize)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes 0-"+strconv.Itoa(patchSize-1)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, bytes.Join([][]byte{patchBody, data1[patchSize:], data2, data3}, []byte("")), object)
		require.Equal(t, partSize*3, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
	})

	t.Run("patch middle of the first part", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
		etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
		etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})

		patchSize := partSize / 2
		patchBody := make([]byte, patchSize)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize/4)+"-"+strconv.Itoa(partSize*3/4-1)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, bytes.Join([][]byte{data1[:partSize/4], patchBody, data1[partSize*3/4:], data2, data3}, []byte("")), object)
		require.Equal(t, partSize*3, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
	})

	t.Run("patch first and second parts", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
		etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
		etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})

		patchSize := partSize / 2
		patchBody := make([]byte, patchSize)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*3/4)+"-"+strconv.Itoa(partSize*5/4-1)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, bytes.Join([][]byte{data1[:partSize*3/4], patchBody, data2[partSize/4:], data3}, []byte("")), object)
		require.Equal(t, partSize*3, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
	})

	t.Run("patch all parts", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
		etag2, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
		etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})

		patchSize := partSize * 2
		patchBody := make([]byte, patchSize)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize/2-1)+"-"+strconv.Itoa(partSize/2+patchSize-2)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, bytes.Join([][]byte{data1[:partSize/2-1], patchBody, data3[partSize/2-1:]}, []byte("")), object)
		require.Equal(t, partSize*3, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
	})

	t.Run("patch all parts and append bytes", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
		etag2, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
		etag3, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})

		patchSize := partSize * 3
		patchBody := make([]byte, patchSize)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize/2)+"-"+strconv.Itoa(partSize/2+patchSize-1)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, bytes.Join([][]byte{data1[:partSize/2], patchBody}, []byte("")), object)
		require.Equal(t, partSize*7/2, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
	})

	t.Run("patch second part", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
		etag2, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
		etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})

		patchBody := make([]byte, partSize)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize)+"-"+strconv.Itoa(partSize*2-1)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, bytes.Join([][]byte{data1, patchBody, data3}, []byte("")), object)
		require.Equal(t, partSize*3, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
	})

	t.Run("patch last part, equal size", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
		etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
		etag3, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})

		patchBody := make([]byte, partSize)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*2)+"-"+strconv.Itoa(partSize*3-1)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, bytes.Join([][]byte{data1, data2, patchBody}, []byte("")), object)
		require.Equal(t, partSize*3, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
	})

	t.Run("patch last part, increase size", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
		etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
		etag3, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})

		patchBody := make([]byte, partSize+1)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*2)+"-"+strconv.Itoa(partSize*3)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, bytes.Join([][]byte{data1, data2, patchBody}, []byte("")), object)
		require.Equal(t, partSize*3+1, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
	})

	t.Run("patch last part with offset and append bytes", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
		etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
		etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})

		patchBody := make([]byte, partSize)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*2+3)+"-"+strconv.Itoa(partSize*3+2)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, bytes.Join([][]byte{data1, data2, data3[:3], patchBody}, []byte("")), object)
		require.Equal(t, partSize*3+3, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
	})

	t.Run("append bytes", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
		etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
		etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})

		patchBody := make([]byte, partSize)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*3)+"-"+strconv.Itoa(partSize*4-1)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, bytes.Join([][]byte{data1, data2, data3, patchBody}, []byte("")), object)
		require.Equal(t, partSize*4, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
	})

	t.Run("patch empty multipart", func(t *testing.T) {
		multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
		etag, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, 0)
		completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag})

		patchBody := make([]byte, partSize)
		_, err := rand.Read(patchBody)
		require.NoError(t, err)

		patchObject(t, tc, bktName, objName, "bytes 0-"+strconv.Itoa(partSize-1)+"/*", patchBody, nil)
		object, header := getObject(tc, bktName, objName)
		contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
		require.NoError(t, err)
		equalDataSlices(t, patchBody, object)
		require.Equal(t, partSize, contentLen)
		require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-1"))
	})
}

func TestPatchWithVersion(t *testing.T) {
	hc := prepareHandlerContextWithMinCache(t)
	bktName, objName := "bucket", "obj"
	createVersionedBucket(hc, bktName)
	objHeader := putObjectContent(hc, bktName, objName, "content")

	putObjectContent(hc, bktName, objName, "some content")

	patchObjectVersion(t, hc, bktName, objName, objHeader.Get(api.AmzVersionID), "bytes 7-14/*", []byte(" updated"))

	res := listObjectsVersions(hc, bktName, "", "", "", "", 3)
	require.False(t, res.IsTruncated)
	require.Len(t, res.Version, 3)

	for _, version := range res.Version {
		content := getObjectVersion(hc, bktName, objName, version.VersionID)
		if version.IsLatest {
			require.Equal(t, []byte("content updated"), content)
			continue
		}
		if version.VersionID == objHeader.Get(api.AmzVersionID) {
			require.Equal(t, []byte("content"), content)
			continue
		}
		require.Equal(t, []byte("some content"), content)
	}
}

func TestPatchEncryptedObject(t *testing.T) {
	tc := prepareHandlerContext(t)
	bktName, objName := "bucket-for-patch-encrypted", "object-for-patch-encrypted"
	createTestBucket(tc, bktName)

	w, r := prepareTestPayloadRequest(tc, bktName, objName, strings.NewReader("object content"))
	setEncryptHeaders(r)
	tc.Handler().PutObjectHandler(w, r)
	assertStatus(t, w, http.StatusOK)

	patchObjectErr(t, tc, bktName, objName, "bytes 2-4/*", []byte("new"), nil, s3errors.ErrInternalError)
}

func TestPatchMissingHeaders(t *testing.T) {
	tc := prepareHandlerContext(t)
	bktName, objName := "bucket-for-patch-missing-headers", "object-for-patch-missing-headers"
	createTestBucket(tc, bktName)

	w, r := prepareTestPayloadRequest(tc, bktName, objName, strings.NewReader("object content"))
	setEncryptHeaders(r)
	tc.Handler().PutObjectHandler(w, r)
	assertStatus(t, w, http.StatusOK)

	w = httptest.NewRecorder()
	r = httptest.NewRequest(http.MethodPatch, defaultURL, strings.NewReader("new"))
	tc.Handler().PatchObjectHandler(w, r)
	assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrMissingContentRange))

	w = httptest.NewRecorder()
	r = httptest.NewRequest(http.MethodPatch, defaultURL, strings.NewReader("new"))
	r.Header.Set(api.ContentRange, "bytes 0-2/*")
	tc.Handler().PatchObjectHandler(w, r)
	assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrMissingContentLength))
}

func TestParsePatchByteRange(t *testing.T) {
	for _, tt := range []struct {
		rng      string
		size     uint64
		expected *layer.RangeParams
		err      bool
	}{
		{
			rng: "bytes 2-7/*",
			expected: &layer.RangeParams{
				Start: 2,
				End:   7,
			},
		},
		{
			rng: "bytes 2-7/3",
			expected: &layer.RangeParams{
				Start: 2,
				End:   7,
			},
		},
		{
			rng:  "bytes 2-/*",
			size: 9,
			expected: &layer.RangeParams{
				Start: 2,
				End:   8,
			},
		},
		{
			rng:  "bytes 2-/3",
			size: 9,
			expected: &layer.RangeParams{
				Start: 2,
				End:   8,
			},
		},
		{
			rng: "",
			err: true,
		},
		{
			rng: "2-7/*",
			err: true,
		},
		{
			rng: "bytes 7-2/*",
			err: true,
		},
		{
			rng: "bytes 2-7",
			err: true,
		},
		{
			rng: "bytes 2/*",
			err: true,
		},
		{
			rng: "bytes a-7/*",
			err: true,
		},
		{
			rng: "bytes 2-a/*",
			err: true,
		},
	} {
		t.Run(fmt.Sprintf("case: %s", tt.rng), func(t *testing.T) {
			rng, err := parsePatchByteRange(tt.rng, tt.size)
			if tt.err {
				require.Error(t, err)
			} else {
				require.NoError(t, err)
				require.Equal(t, tt.expected.Start, rng.Start)
				require.Equal(t, tt.expected.End, rng.End)
			}
		})
	}
}

func patchObject(t *testing.T, tc *handlerContext, bktName, objName, rng string, payload []byte, headers map[string]string) *PatchObjectResult {
	w := patchObjectBase(tc, bktName, objName, "", rng, payload, headers)
	assertStatus(t, w, http.StatusOK)

	result := &PatchObjectResult{}
	err := xml.NewDecoder(w.Result().Body).Decode(result)
	require.NoError(t, err)
	return result
}

func patchObjectVersion(t *testing.T, tc *handlerContext, bktName, objName, version, rng string, payload []byte) *PatchObjectResult {
	w := patchObjectBase(tc, bktName, objName, version, rng, payload, nil)
	assertStatus(t, w, http.StatusOK)

	result := &PatchObjectResult{}
	err := xml.NewDecoder(w.Result().Body).Decode(result)
	require.NoError(t, err)
	return result
}

func patchObjectErr(t *testing.T, tc *handlerContext, bktName, objName, rng string, payload []byte, headers map[string]string, code s3errors.ErrorCode) {
	w := patchObjectBase(tc, bktName, objName, "", rng, payload, headers)
	assertS3Error(t, w, s3errors.GetAPIError(code))
}

func patchObjectBase(tc *handlerContext, bktName, objName, version, rng string, payload []byte, headers map[string]string) *httptest.ResponseRecorder {
	query := make(url.Values)
	if len(version) > 0 {
		query.Add(api.QueryVersionID, version)
	}

	w, r := prepareTestRequestWithQuery(tc, bktName, objName, query, payload)
	r.Header.Set(api.ContentRange, rng)
	r.Header.Set(api.ContentLength, strconv.Itoa(len(payload)))
	for k, v := range headers {
		r.Header.Set(k, v)
	}

	tc.Handler().PatchObjectHandler(w, r)
	return w
}