[#106] Add chunk uploading

Signed-off-by: Artem Tataurov <a.tataurov@yadro.com>

api/handler/s3reader.go

@@ -0,0 +1,224 @@
+package handler
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net/http"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
+	v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
+	errs "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+)
+
+const (
+	chunkSignatureHeader = "chunk-signature="
+	maxChunkSize         = 16 << 20
+)
+
+type (
+	s3ChunkReader struct {
+		reader       *bufio.Reader
+		streamSigner *v4.StreamSigner
+
+		requestTime time.Time
+		buffer      []byte
+		offset      int
+		err         error
+	}
+)
+
+var (
+	errGiantChunk               = errors.New("chunk too big: choose chunk size <= 16MiB")
+	errMalformedChunkedEncoding = errors.New("malformed chunked encoding")
+)
+
+func (c *s3ChunkReader) Close() (err error) {
+	return nil
+}
+
+func (c *s3ChunkReader) Read(buf []byte) (num int, err error) {
+	if c.offset > 0 {
+		num = copy(buf, c.buffer[c.offset:])
+		if num == len(buf) {
+			c.offset += num
+			return num, nil
+		}
+		c.offset = 0
+		buf = buf[num:]
+	}
+
+	var size int
+	for {
+		b, err := c.reader.ReadByte()
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		if err != nil {
+			c.err = err
+			return num, c.err
+		}
+		if b == ';' { // separating character
+			break
+		}
+
+		// Manually deserialize the size since AWS specified
+		// the chunk size to be of variable width. In particular,
+		// a size of 16 is encoded as `10` while a size of 64 KB
+		// is `10000`.
+		switch {
+		case b >= '0' && b <= '9':
+			size = size<<4 | int(b-'0')
+		case b >= 'a' && b <= 'f':
+			size = size<<4 | int(b-('a'-10))
+		case b >= 'A' && b <= 'F':
+			size = size<<4 | int(b-('A'-10))
+		default:
+			c.err = errMalformedChunkedEncoding
+			return num, c.err
+		}
+		if size > maxChunkSize {
+			c.err = errGiantChunk
+			return num, c.err
+		}
+	}
+
+	// Now, we read the signature of the following payload and expect:
+	//   chunk-signature=" + <signature-as-hex> + "\r\n"
+	//
+	// The signature is 64 bytes long (hex-encoded SHA256 hash) and
+	// starts with a 16 byte header: len("chunk-signature=") + 64 == 80.
+	var signature [80]byte
+	_, err = io.ReadFull(c.reader, signature[:])
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	}
+	if err != nil {
+		c.err = err
+		return num, c.err
+	}
+	if !bytes.HasPrefix(signature[:], []byte(chunkSignatureHeader)) {
+		c.err = errMalformedChunkedEncoding
+		return num, c.err
+	}
+	b, err := c.reader.ReadByte()
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	}
+	if err != nil {
+		c.err = err
+		return num, c.err
+	}
+	if b != '\r' {
+		c.err = errMalformedChunkedEncoding
+		return num, c.err
+	}
+	b, err = c.reader.ReadByte()
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	}
+	if err != nil {
+		c.err = err
+		return num, c.err
+	}
+	if b != '\n' {
+		c.err = errMalformedChunkedEncoding
+		return num, c.err
+	}
+
+	if cap(c.buffer) < size {
+		c.buffer = make([]byte, size)
+	} else {
+		c.buffer = c.buffer[:size]
+	}
+
+	// Now, we read the payload and compute its SHA-256 hash.
+	_, err = io.ReadFull(c.reader, c.buffer)
+	if err == io.EOF && size != 0 {
+		err = io.ErrUnexpectedEOF
+	}
+	if err != nil && err != io.EOF {
+		c.err = err
+		return num, c.err
+	}
+	b, err = c.reader.ReadByte()
+	if b != '\r' || err != nil {
+		c.err = errMalformedChunkedEncoding
+		return num, c.err
+	}
+	b, err = c.reader.ReadByte()
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	}
+	if err != nil {
+		c.err = err
+		return num, c.err
+	}
+	if b != '\n' {
+		c.err = errMalformedChunkedEncoding
+		return num, c.err
+	}
+
+	// Once we have read the entire chunk successfully, we verify
+	// that the received signature matches our computed signature.
+
+	calculatedSignature, err := c.streamSigner.GetSignature(nil, c.buffer, c.requestTime)
+	if err != nil {
+		c.err = err
+		return num, c.err
+	}
+	if string(signature[16:]) != hex.EncodeToString(calculatedSignature) {
+		c.err = errs.GetAPIError(errs.ErrSignatureDoesNotMatch)
+		return num, c.err
+	}
+
+	// If the chunk size is zero we return io.EOF. As specified by AWS,
+	// only the last chunk is zero-sized.
+	if size == 0 {
+		c.err = io.EOF
+		return num, c.err
+	}
+
+	c.offset = copy(buf, c.buffer)
+	num += c.offset
+	return num, err
+}
+
+func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, error) {
+	// Expecting to refactor this in future:
+	// https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/issues/137
+	box, ok := req.Context().Value(api.BoxData).(*accessbox.Box)
+	if !ok {
+		return nil, errs.GetAPIError(errs.ErrAuthorizationHeaderMalformed)
+	}
+
+	authHeaders, ok := req.Context().Value(api.AuthHeaders).(*auth.AuthHeader)
+	if !ok {
+		return nil, errs.GetAPIError(errs.ErrAuthorizationHeaderMalformed)
+	}
+
+	currentCredentials := credentials.NewStaticCredentials(authHeaders.AccessKeyID, box.Gate.AccessKey, "")
+	seed, err := hex.DecodeString(authHeaders.SignatureV4)
+	if err != nil {
+		return nil, errs.GetAPIError(errs.ErrSignatureDoesNotMatch)
+	}
+
+	reqTime, ok := req.Context().Value(api.ClientTime).(time.Time)
+	if !ok {
+		return nil, errs.GetAPIError(errs.ErrMalformedDate)
+	}
+	newStreamSigner := v4.NewStreamSigner(authHeaders.Region, "s3", seed, currentCredentials)
+
+	return &s3ChunkReader{
+		reader:       bufio.NewReader(req.Body),
+		streamSigner: newStreamSigner,
+		requestTime:  reqTime,
+		buffer:       make([]byte, 64*1024),
+	}, nil
+}