forked from TrueCloudLab/frostfs-s3-gw
[#595] Move decrypter to separate package
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
parent
d824db7f69
commit
94a6a55919
14 changed files with 445 additions and 427 deletions
|
@ -36,7 +36,6 @@ type (
|
||||||
CID cid.ID
|
CID cid.ID
|
||||||
IsDir bool
|
IsDir bool
|
||||||
IsDeleteMarker bool
|
IsDeleteMarker bool
|
||||||
EncryptionInfo EncryptionInfo
|
|
||||||
|
|
||||||
Bucket string
|
Bucket string
|
||||||
Name string
|
Name string
|
||||||
|
@ -48,14 +47,6 @@ type (
|
||||||
Headers map[string]string
|
Headers map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// EncryptionInfo store parsed object encryption headers.
|
|
||||||
EncryptionInfo struct {
|
|
||||||
Enabled bool
|
|
||||||
Algorithm string
|
|
||||||
HMACKey string
|
|
||||||
HMACSalt string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NotificationInfo store info to send s3 notification.
|
// NotificationInfo store info to send s3 notification.
|
||||||
NotificationInfo struct {
|
NotificationInfo struct {
|
||||||
Name string
|
Name string
|
||||||
|
@ -122,11 +113,6 @@ func (o *ObjectInfo) Address() oid.Address {
|
||||||
return addr
|
return addr
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsEncrypted returns true if object is encrypted.
|
|
||||||
func (o ObjectInfo) IsEncrypted() bool {
|
|
||||||
return o.EncryptionInfo.Enabled
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b BucketSettings) Unversioned() bool {
|
func (b BucketSettings) Unversioned() bool {
|
||||||
return b.Versioning == VersioningUnversioned
|
return b.Versioning == VersioningUnversioned
|
||||||
}
|
}
|
||||||
|
|
|
@ -140,7 +140,6 @@ const (
|
||||||
ErrSSEMultipartEncrypted
|
ErrSSEMultipartEncrypted
|
||||||
ErrSSEEncryptedObject
|
ErrSSEEncryptedObject
|
||||||
ErrInvalidEncryptionParameters
|
ErrInvalidEncryptionParameters
|
||||||
ErrInvalidSSECustomerAlgorithm
|
|
||||||
ErrInvalidEncryptionAlgorithm
|
ErrInvalidEncryptionAlgorithm
|
||||||
ErrInvalidSSECustomerKey
|
ErrInvalidSSECustomerKey
|
||||||
ErrMissingSSECustomerKey
|
ErrMissingSSECustomerKey
|
||||||
|
@ -1006,12 +1005,6 @@ var errorCodes = errorCodeMap{
|
||||||
Description: "The encryption parameters are not applicable to this object.",
|
Description: "The encryption parameters are not applicable to this object.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrInvalidSSECustomerAlgorithm: {
|
|
||||||
ErrCode: ErrInvalidSSECustomerAlgorithm,
|
|
||||||
Code: "InvalidArgument",
|
|
||||||
Description: "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm.",
|
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
|
||||||
},
|
|
||||||
ErrInvalidEncryptionAlgorithm: {
|
ErrInvalidEncryptionAlgorithm: {
|
||||||
ErrCode: ErrInvalidEncryptionAlgorithm,
|
ErrCode: ErrInvalidEncryptionAlgorithm,
|
||||||
Code: "InvalidArgument",
|
Code: "InvalidArgument",
|
||||||
|
|
|
@ -94,13 +94,13 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
|
||||||
}
|
}
|
||||||
info := extendedInfo.ObjectInfo
|
info := extendedInfo.ObjectInfo
|
||||||
|
|
||||||
encryption, err := h.formEncryptionParams(r.Header)
|
encryptionParams, err := h.formEncryptionParams(r.Header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = encryption.MatchObjectEncryption(info.EncryptionInfo); err != nil {
|
if err = encryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(info.Headers)); err != nil {
|
||||||
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -214,23 +214,14 @@ func formUploadAttributes(info *data.ObjectInfo, maxParts, marker int) (*ObjectP
|
||||||
partInfos := strings.Split(completedParts, ",")
|
partInfos := strings.Split(completedParts, ",")
|
||||||
parts := make([]Part, len(partInfos))
|
parts := make([]Part, len(partInfos))
|
||||||
for i, p := range partInfos {
|
for i, p := range partInfos {
|
||||||
// partInfo[0] -- part number, partInfo[1] -- part size, partInfo[2] -- checksum
|
part, err := layer.ParseCompletedPartHeader(p)
|
||||||
partInfo := strings.Split(p, "-")
|
|
||||||
if len(partInfo) != 3 {
|
|
||||||
return nil, fmt.Errorf("invalid completed parts header")
|
|
||||||
}
|
|
||||||
num, err := strconv.Atoi(partInfo[0])
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("invalid competed part: %w", err)
|
||||||
}
|
|
||||||
size, err := strconv.Atoi(partInfo[1])
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
parts[i] = Part{
|
parts[i] = Part{
|
||||||
PartNumber: num,
|
PartNumber: part.PartNumber,
|
||||||
Size: size,
|
Size: int(part.Size),
|
||||||
ChecksumSHA256: partInfo[2],
|
ChecksumSHA256: part.ETag,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -96,13 +96,13 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
encryption, err := h.formEncryptionParams(r.Header)
|
encryptionParams, err := h.formEncryptionParams(r.Header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = encryption.MatchObjectEncryption(objInfo.EncryptionInfo); err != nil {
|
if err = encryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(objInfo.Headers)); err != nil {
|
||||||
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -128,7 +128,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
DstObject: reqInfo.ObjectName,
|
DstObject: reqInfo.ObjectName,
|
||||||
SrcSize: objInfo.Size,
|
SrcSize: objInfo.Size,
|
||||||
Header: metadata,
|
Header: metadata,
|
||||||
Encryption: encryption,
|
Encryption: encryptionParams,
|
||||||
}
|
}
|
||||||
|
|
||||||
settings, err := h.obj.GetBucketSettings(r.Context(), dstBktInfo)
|
settings, err := h.obj.GetBucketSettings(r.Context(), dstBktInfo)
|
||||||
|
@ -186,7 +186,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if encryption.Enabled() {
|
if encryptionParams.Enabled() {
|
||||||
addSSECHeaders(w.Header(), r.Header)
|
addSSECHeaders(w.Header(), r.Header)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -84,7 +84,7 @@ func writeHeaders(h http.Header, requestHeader http.Header, extendedInfo *data.E
|
||||||
}
|
}
|
||||||
h.Set(api.LastModified, info.Created.UTC().Format(http.TimeFormat))
|
h.Set(api.LastModified, info.Created.UTC().Format(http.TimeFormat))
|
||||||
|
|
||||||
if info.IsEncrypted() {
|
if len(info.Headers[layer.AttributeEncryptionAlgorithm]) > 0 {
|
||||||
h.Set(api.ContentLength, info.Headers[layer.AttributeDecryptedSize])
|
h.Set(api.ContentLength, info.Headers[layer.AttributeDecryptedSize])
|
||||||
addSSECHeaders(h, requestHeader)
|
addSSECHeaders(h, requestHeader)
|
||||||
} else {
|
} else {
|
||||||
|
@ -150,19 +150,19 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
encryption, err := h.formEncryptionParams(r.Header)
|
encryptionParams, err := h.formEncryptionParams(r.Header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = encryption.MatchObjectEncryption(info.EncryptionInfo); err != nil {
|
if err = encryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(info.Headers)); err != nil {
|
||||||
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
fullSize := info.Size
|
fullSize := info.Size
|
||||||
if encryption.Enabled() {
|
if encryptionParams.Enabled() {
|
||||||
if fullSize, err = strconv.ParseInt(info.Headers[layer.AttributeDecryptedSize], 10, 64); err != nil {
|
if fullSize, err = strconv.ParseInt(info.Headers[layer.AttributeDecryptedSize], 10, 64); err != nil {
|
||||||
h.logAndSendError(w, "invalid decrypted size header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
|
h.logAndSendError(w, "invalid decrypted size header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
|
||||||
return
|
return
|
||||||
|
@ -213,7 +213,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
Writer: w,
|
Writer: w,
|
||||||
Range: params,
|
Range: params,
|
||||||
BucketInfo: bktInfo,
|
BucketInfo: bktInfo,
|
||||||
Encryption: encryption,
|
Encryption: encryptionParams,
|
||||||
}
|
}
|
||||||
if err = h.obj.GetObject(r.Context(), getParams); err != nil {
|
if err = h.obj.GetObject(r.Context(), getParams); err != nil {
|
||||||
h.logAndSendError(w, "could not get object", reqInfo, err)
|
h.logAndSendError(w, "could not get object", reqInfo, err)
|
||||||
|
|
|
@ -53,13 +53,13 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
info := extendedInfo.ObjectInfo
|
info := extendedInfo.ObjectInfo
|
||||||
|
|
||||||
encryption, err := h.formEncryptionParams(r.Header)
|
encryptionParams, err := h.formEncryptionParams(r.Header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = encryption.MatchObjectEncryption(info.EncryptionInfo); err != nil {
|
if err = encryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(info.Headers)); err != nil {
|
||||||
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -327,7 +327,7 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = p.Info.Encryption.MatchObjectEncryption(srcInfo.EncryptionInfo); err != nil {
|
if err = p.Info.Encryption.MatchObjectEncryption(layer.FormEncryptionInfo(srcInfo.Headers)); err != nil {
|
||||||
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,6 +21,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||||
|
@ -297,7 +298,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
api.WriteSuccessResponseHeadersOnly(w)
|
api.WriteSuccessResponseHeadersOnly(w)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h handler) formEncryptionParams(header http.Header) (enc layer.EncryptionParams, err error) {
|
func (h handler) formEncryptionParams(header http.Header) (enc encryption.Params, err error) {
|
||||||
sseCustomerAlgorithm := header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
|
sseCustomerAlgorithm := header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
|
||||||
sseCustomerKey := header.Get(api.AmzServerSideEncryptionCustomerKey)
|
sseCustomerKey := header.Get(api.AmzServerSideEncryptionCustomerKey)
|
||||||
sseCustomerKeyMD5 := header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
|
sseCustomerKeyMD5 := header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
|
||||||
|
@ -333,10 +334,7 @@ func (h handler) formEncryptionParams(header http.Header) (enc layer.EncryptionP
|
||||||
return enc, errors.GetAPIError(errors.ErrSSECustomerKeyMD5Mismatch)
|
return enc, errors.GetAPIError(errors.ErrSSECustomerKeyMD5Mismatch)
|
||||||
}
|
}
|
||||||
|
|
||||||
var aesKey layer.AES256Key
|
return encryption.NewParams(key)
|
||||||
copy(aesKey[:], key)
|
|
||||||
|
|
||||||
return layer.NewEncryptionParams(aesKey), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
344
api/layer/encryption/encryption.go
Normal file
344
api/layer/encryption/encryption.go
Normal file
|
@ -0,0 +1,344 @@
|
||||||
|
package encryption
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
errorsStd "errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"github.com/minio/sio"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Params contains encryption key info.
|
||||||
|
type Params struct {
|
||||||
|
enabled bool
|
||||||
|
customerKey []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObjectEncryption stores parsed object encryption headers.
|
||||||
|
type ObjectEncryption struct {
|
||||||
|
Enabled bool
|
||||||
|
Algorithm string
|
||||||
|
HMACKey string
|
||||||
|
HMACSalt string
|
||||||
|
}
|
||||||
|
|
||||||
|
type encryptedPart struct {
|
||||||
|
size uint64
|
||||||
|
encryptedSize uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Range stores payload interval.
|
||||||
|
type Range struct {
|
||||||
|
Start uint64
|
||||||
|
End uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decrypter allows decrypt payload of encrypted object.
|
||||||
|
type Decrypter struct {
|
||||||
|
reader io.Reader
|
||||||
|
decReader io.Reader
|
||||||
|
parts []encryptedPart
|
||||||
|
currentPart int
|
||||||
|
encryption Params
|
||||||
|
|
||||||
|
rangeParam *Range
|
||||||
|
|
||||||
|
partDataRemain uint64
|
||||||
|
encPartRangeLen uint64
|
||||||
|
|
||||||
|
seqNumber uint64
|
||||||
|
decLen uint64
|
||||||
|
skipLen uint64
|
||||||
|
|
||||||
|
ln uint64
|
||||||
|
off uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
blockSize = 1 << 16 // 64KB
|
||||||
|
fullBlockSize = blockSize + 32
|
||||||
|
aes256KeySize = 32
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewParams creates new params to encrypt with provided key.
|
||||||
|
func NewParams(key []byte) (Params, error) {
|
||||||
|
var p Params
|
||||||
|
if len(key) != aes256KeySize {
|
||||||
|
return p, fmt.Errorf("invalid key size: %d", len(key))
|
||||||
|
}
|
||||||
|
p.enabled = true
|
||||||
|
p.customerKey = make([]byte, aes256KeySize)
|
||||||
|
copy(p.customerKey, key)
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Key returns encryption key as slice.
|
||||||
|
func (p Params) Key() []byte {
|
||||||
|
return p.customerKey[:]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enabled returns true if key isn't empty.
|
||||||
|
func (p Params) Enabled() bool {
|
||||||
|
return p.enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
// HMAC computes salted HMAC.
|
||||||
|
func (p Params) HMAC() ([]byte, []byte, error) {
|
||||||
|
mac := hmac.New(sha256.New, p.Key())
|
||||||
|
|
||||||
|
salt := make([]byte, 16)
|
||||||
|
if _, err := rand.Read(salt); err != nil {
|
||||||
|
return nil, nil, errorsStd.New("failed to init create salt")
|
||||||
|
}
|
||||||
|
|
||||||
|
mac.Write(salt)
|
||||||
|
return mac.Sum(nil), salt, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MatchObjectEncryption checks if encryption params are valid for provided object.
|
||||||
|
func (p Params) MatchObjectEncryption(encInfo ObjectEncryption) error {
|
||||||
|
if p.Enabled() != encInfo.Enabled {
|
||||||
|
return errorsStd.New("invalid encryption view")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !encInfo.Enabled {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
hmacSalt, err := hex.DecodeString(encInfo.HMACSalt)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid hmacSalt '%s': %w", encInfo.HMACSalt, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
hmacKey, err := hex.DecodeString(encInfo.HMACKey)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid hmacKey '%s': %w", encInfo.HMACKey, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mac := hmac.New(sha256.New, p.Key())
|
||||||
|
mac.Write(hmacSalt)
|
||||||
|
expectedHmacKey := mac.Sum(nil)
|
||||||
|
if !bytes.Equal(expectedHmacKey, hmacKey) {
|
||||||
|
return errorsStd.New("mismatched hmac key")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMultipartDecrypter creates new decrypted that can decrypt multipart object
|
||||||
|
// that contains concatenation of encrypted parts.
|
||||||
|
func NewMultipartDecrypter(p Params, decryptedObjectSize uint64, partsSizes []uint64, r *Range) (*Decrypter, error) {
|
||||||
|
parts := make([]encryptedPart, len(partsSizes))
|
||||||
|
|
||||||
|
for i, size := range partsSizes {
|
||||||
|
encPartSize, err := sio.EncryptedSize(size)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("compute encrypted size: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
parts[i] = encryptedPart{
|
||||||
|
size: size,
|
||||||
|
encryptedSize: encPartSize,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rangeParam := r
|
||||||
|
if rangeParam == nil {
|
||||||
|
rangeParam = &Range{
|
||||||
|
End: decryptedObjectSize - 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return newDecrypter(p, parts, rangeParam)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDecrypter creates decrypter for regular encrypted object.
|
||||||
|
func NewDecrypter(p Params, encryptedObjectSize uint64, r *Range) (*Decrypter, error) {
|
||||||
|
decSize, err := sio.DecryptedSize(encryptedObjectSize)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("compute decrypted size: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := []encryptedPart{{
|
||||||
|
size: decSize,
|
||||||
|
encryptedSize: encryptedObjectSize,
|
||||||
|
}}
|
||||||
|
|
||||||
|
return newDecrypter(p, parts, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newDecrypter(p Params, parts []encryptedPart, r *Range) (*Decrypter, error) {
|
||||||
|
if !p.Enabled() {
|
||||||
|
return nil, errorsStd.New("couldn't create decrypter with disabled encryption")
|
||||||
|
}
|
||||||
|
|
||||||
|
if r != nil && r.Start > r.End {
|
||||||
|
return nil, fmt.Errorf("invalid range: %d %d", r.Start, r.End)
|
||||||
|
}
|
||||||
|
|
||||||
|
decReader := &Decrypter{
|
||||||
|
parts: parts,
|
||||||
|
rangeParam: r,
|
||||||
|
encryption: p,
|
||||||
|
}
|
||||||
|
|
||||||
|
decReader.initRangeParams()
|
||||||
|
|
||||||
|
return decReader, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecryptedLength is actual (decrypted) length of data.
|
||||||
|
func (d Decrypter) DecryptedLength() uint64 {
|
||||||
|
return d.decLen
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncryptedLength is size of encrypted data that should be read for successful decryption.
|
||||||
|
func (d Decrypter) EncryptedLength() uint64 {
|
||||||
|
return d.ln
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncryptedOffset is offset of encrypted payload for successful decryption.
|
||||||
|
func (d Decrypter) EncryptedOffset() uint64 {
|
||||||
|
return d.off
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Decrypter) initRangeParams() {
|
||||||
|
d.partDataRemain = d.parts[d.currentPart].size
|
||||||
|
d.encPartRangeLen = d.parts[d.currentPart].encryptedSize
|
||||||
|
if d.rangeParam == nil {
|
||||||
|
d.decLen = d.partDataRemain
|
||||||
|
d.ln = d.encPartRangeLen
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
start, end := d.rangeParam.Start, d.rangeParam.End
|
||||||
|
|
||||||
|
var sum, encSum uint64
|
||||||
|
var partStart int
|
||||||
|
for i, part := range d.parts {
|
||||||
|
if start < sum+part.size {
|
||||||
|
partStart = i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
sum += part.size
|
||||||
|
encSum += part.encryptedSize
|
||||||
|
}
|
||||||
|
|
||||||
|
d.skipLen = (start - sum) % blockSize
|
||||||
|
d.seqNumber = (start - sum) / blockSize
|
||||||
|
encOffPart := d.seqNumber * fullBlockSize
|
||||||
|
d.off = encSum + encOffPart
|
||||||
|
d.encPartRangeLen = d.encPartRangeLen - encOffPart
|
||||||
|
d.partDataRemain = d.partDataRemain + sum - start
|
||||||
|
|
||||||
|
var partEnd int
|
||||||
|
for i, part := range d.parts[partStart:] {
|
||||||
|
index := partStart + i
|
||||||
|
if end < sum+part.size {
|
||||||
|
partEnd = index
|
||||||
|
break
|
||||||
|
}
|
||||||
|
sum += part.size
|
||||||
|
encSum += part.encryptedSize
|
||||||
|
}
|
||||||
|
|
||||||
|
payloadPartEnd := (end - sum) / blockSize
|
||||||
|
endEnc := encSum + (payloadPartEnd+1)*fullBlockSize
|
||||||
|
|
||||||
|
endPartEnc := encSum + d.parts[partEnd].encryptedSize
|
||||||
|
if endPartEnc < endEnc {
|
||||||
|
endEnc = endPartEnc
|
||||||
|
}
|
||||||
|
d.ln = endEnc - d.off
|
||||||
|
d.decLen = end - start + 1
|
||||||
|
|
||||||
|
if d.ln < d.encPartRangeLen {
|
||||||
|
d.encPartRangeLen = d.ln
|
||||||
|
}
|
||||||
|
if d.decLen < d.partDataRemain {
|
||||||
|
d.partDataRemain = d.decLen
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Decrypter) updateRangeParams() {
|
||||||
|
d.partDataRemain = d.parts[d.currentPart].size
|
||||||
|
d.encPartRangeLen = d.parts[d.currentPart].encryptedSize
|
||||||
|
d.seqNumber = 0
|
||||||
|
d.skipLen = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read implements io.Reader.
|
||||||
|
func (d *Decrypter) Read(p []byte) (int, error) {
|
||||||
|
if uint64(len(p)) < d.partDataRemain {
|
||||||
|
n, err := d.decReader.Read(p)
|
||||||
|
if err != nil {
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
d.partDataRemain -= uint64(n)
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
n1, err := io.ReadFull(d.decReader, p[:d.partDataRemain])
|
||||||
|
if err != nil {
|
||||||
|
return n1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
d.currentPart++
|
||||||
|
if d.currentPart == len(d.parts) {
|
||||||
|
return n1, io.EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
d.updateRangeParams()
|
||||||
|
|
||||||
|
err = d.initNextDecReader()
|
||||||
|
if err != nil {
|
||||||
|
return n1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
n2, err := d.decReader.Read(p[n1:])
|
||||||
|
if err != nil {
|
||||||
|
return n1 + n2, err
|
||||||
|
}
|
||||||
|
|
||||||
|
d.partDataRemain -= uint64(n2)
|
||||||
|
|
||||||
|
return n1 + n2, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetReader sets encrypted payload reader that should be decrypted.
|
||||||
|
// Must be invoked before any read.
|
||||||
|
func (d *Decrypter) SetReader(r io.Reader) error {
|
||||||
|
d.reader = r
|
||||||
|
return d.initNextDecReader()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Decrypter) initNextDecReader() error {
|
||||||
|
if d.reader == nil {
|
||||||
|
return errorsStd.New("reader isn't set")
|
||||||
|
}
|
||||||
|
|
||||||
|
r, err := sio.DecryptReader(io.LimitReader(d.reader, int64(d.encPartRangeLen)),
|
||||||
|
sio.Config{
|
||||||
|
MinVersion: sio.Version20,
|
||||||
|
SequenceNumber: uint32(d.seqNumber),
|
||||||
|
Key: d.encryption.Key(),
|
||||||
|
CipherSuites: []byte{sio.AES_256_GCM},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("couldn't create decrypter: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if d.skipLen > 0 {
|
||||||
|
if _, err = io.CopyN(io.Discard, r, int64(d.skipLen)); err != nil {
|
||||||
|
return fmt.Errorf("couldn't skip some bytes: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d.decReader = r
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -1,11 +1,10 @@
|
||||||
package layer
|
package encryption
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"strconv"
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -13,19 +12,20 @@ const (
|
||||||
aes256Key = "1234567890qwertyuiopasdfghjklzxc"
|
aes256Key = "1234567890qwertyuiopasdfghjklzxc"
|
||||||
)
|
)
|
||||||
|
|
||||||
func getAES256Key() AES256Key {
|
func getAES256Key() []byte {
|
||||||
var key AES256Key
|
key := make([]byte, 32)
|
||||||
copy(key[:], aes256Key)
|
copy(key[:], aes256Key)
|
||||||
return key
|
return key
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHMAC(t *testing.T) {
|
func TestHMAC(t *testing.T) {
|
||||||
encParam := NewEncryptionParams(getAES256Key())
|
encParam, err := NewParams(getAES256Key())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
hmacKey, hmacSalt, err := encParam.HMAC()
|
hmacKey, hmacSalt, err := encParam.HMAC()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
encInfo := data.EncryptionInfo{
|
encInfo := ObjectEncryption{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
Algorithm: "",
|
Algorithm: "",
|
||||||
HMACKey: hex.EncodeToString(hmacKey),
|
HMACKey: hex.EncodeToString(hmacKey),
|
||||||
|
@ -44,33 +44,34 @@ const (
|
||||||
encPartSize = 5245440 // partSize + enc headers
|
encPartSize = 5245440 // partSize + enc headers
|
||||||
)
|
)
|
||||||
|
|
||||||
func getDecrypter() *decrypter {
|
func getDecrypter(t *testing.T) *Decrypter {
|
||||||
parts := make([]EncryptedPart, partNum)
|
parts := make([]encryptedPart, partNum)
|
||||||
for i := range parts {
|
for i := range parts {
|
||||||
parts[i] = EncryptedPart{
|
parts[i] = encryptedPart{
|
||||||
Part: Part{
|
size: partSize,
|
||||||
PartNumber: i + 1,
|
encryptedSize: encPartSize,
|
||||||
Size: int64(partSize),
|
|
||||||
},
|
|
||||||
EncryptedSize: encPartSize,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &decrypter{
|
|
||||||
|
params, err := NewParams(getAES256Key())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return &Decrypter{
|
||||||
parts: parts,
|
parts: parts,
|
||||||
encryption: NewEncryptionParams(getAES256Key()),
|
encryption: params,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDecrypterInitParams(t *testing.T) {
|
func TestDecrypterInitParams(t *testing.T) {
|
||||||
decReader := getDecrypter()
|
decReader := getDecrypter(t)
|
||||||
|
|
||||||
for i, tc := range []struct {
|
for i, tc := range []struct {
|
||||||
rng *RangeParams
|
rng *Range
|
||||||
expSkipLen, expLn, expOff, expSeqNumber uint64
|
expSkipLen, expLn, expOff, expSeqNumber uint64
|
||||||
expDecLen, expDataRemain, expEncPartRange int64
|
expDecLen, expDataRemain, expEncPartRange uint64
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
rng: &RangeParams{End: objSize - 1},
|
rng: &Range{End: objSize - 1},
|
||||||
expSkipLen: 0,
|
expSkipLen: 0,
|
||||||
expLn: encObjSize,
|
expLn: encObjSize,
|
||||||
expOff: 0,
|
expOff: 0,
|
||||||
|
@ -80,7 +81,7 @@ func TestDecrypterInitParams(t *testing.T) {
|
||||||
expEncPartRange: encPartSize,
|
expEncPartRange: encPartSize,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
rng: &RangeParams{End: 999999},
|
rng: &Range{End: 999999},
|
||||||
expSkipLen: 0,
|
expSkipLen: 0,
|
||||||
expLn: 1049088,
|
expLn: 1049088,
|
||||||
expOff: 0,
|
expOff: 0,
|
||||||
|
@ -90,7 +91,7 @@ func TestDecrypterInitParams(t *testing.T) {
|
||||||
expEncPartRange: 1049088,
|
expEncPartRange: 1049088,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
rng: &RangeParams{Start: 1000000, End: 1999999},
|
rng: &Range{Start: 1000000, End: 1999999},
|
||||||
expSkipLen: 16960,
|
expSkipLen: 16960,
|
||||||
expLn: 1049088,
|
expLn: 1049088,
|
||||||
expOff: 983520,
|
expOff: 983520,
|
|
@ -1,14 +1,9 @@
|
||||||
package layer
|
package layer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"crypto/ecdsa"
|
"crypto/ecdsa"
|
||||||
"crypto/hmac"
|
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
errorsStd "errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
@ -16,7 +11,8 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/sio"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
@ -89,7 +85,7 @@ type (
|
||||||
ObjectInfo *data.ObjectInfo
|
ObjectInfo *data.ObjectInfo
|
||||||
BucketInfo *data.BucketInfo
|
BucketInfo *data.BucketInfo
|
||||||
Writer io.Writer
|
Writer io.Writer
|
||||||
Encryption EncryptionParams
|
Encryption encryption.Params
|
||||||
}
|
}
|
||||||
|
|
||||||
// HeadObjectParams stores object head request parameters.
|
// HeadObjectParams stores object head request parameters.
|
||||||
|
@ -113,14 +109,6 @@ type (
|
||||||
End uint64
|
End uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// AES256Key is a key for encryption.
|
|
||||||
AES256Key [32]byte
|
|
||||||
|
|
||||||
EncryptionParams struct {
|
|
||||||
enabled bool
|
|
||||||
customerKey AES256Key
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutObjectParams stores object put request parameters.
|
// PutObjectParams stores object put request parameters.
|
||||||
PutObjectParams struct {
|
PutObjectParams struct {
|
||||||
BktInfo *data.BucketInfo
|
BktInfo *data.BucketInfo
|
||||||
|
@ -129,7 +117,7 @@ type (
|
||||||
Reader io.Reader
|
Reader io.Reader
|
||||||
Header map[string]string
|
Header map[string]string
|
||||||
Lock *data.ObjectLock
|
Lock *data.ObjectLock
|
||||||
Encryption EncryptionParams
|
Encryption encryption.Params
|
||||||
}
|
}
|
||||||
|
|
||||||
DeleteObjectParams struct {
|
DeleteObjectParams struct {
|
||||||
|
@ -160,7 +148,7 @@ type (
|
||||||
Header map[string]string
|
Header map[string]string
|
||||||
Range *RangeParams
|
Range *RangeParams
|
||||||
Lock *data.ObjectLock
|
Lock *data.ObjectLock
|
||||||
Encryption EncryptionParams
|
Encryption encryption.Params
|
||||||
}
|
}
|
||||||
// CreateBucketParams stores bucket create request parameters.
|
// CreateBucketParams stores bucket create request parameters.
|
||||||
CreateBucketParams struct {
|
CreateBucketParams struct {
|
||||||
|
@ -285,72 +273,6 @@ func (f MsgHandlerFunc) HandleMessage(ctx context.Context, msg *nats.Msg) error
|
||||||
return f(ctx, msg)
|
return f(ctx, msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewEncryptionParams create new params to encrypt with provided key.
|
|
||||||
func NewEncryptionParams(key AES256Key) EncryptionParams {
|
|
||||||
return EncryptionParams{
|
|
||||||
enabled: true,
|
|
||||||
customerKey: key,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Key returns encryption key as slice.
|
|
||||||
func (p EncryptionParams) Key() []byte {
|
|
||||||
return p.customerKey[:]
|
|
||||||
}
|
|
||||||
|
|
||||||
// AESKey returns encryption key.
|
|
||||||
func (p EncryptionParams) AESKey() AES256Key {
|
|
||||||
return p.customerKey
|
|
||||||
}
|
|
||||||
|
|
||||||
// Enabled returns true if key isn't empty.
|
|
||||||
func (p EncryptionParams) Enabled() bool {
|
|
||||||
return p.enabled
|
|
||||||
}
|
|
||||||
|
|
||||||
// HMAC compute salted HMAC.
|
|
||||||
func (p EncryptionParams) HMAC() ([]byte, []byte, error) {
|
|
||||||
mac := hmac.New(sha256.New, p.Key())
|
|
||||||
|
|
||||||
salt := make([]byte, 16)
|
|
||||||
if _, err := rand.Read(salt); err != nil {
|
|
||||||
return nil, nil, errorsStd.New("failed to init create salt")
|
|
||||||
}
|
|
||||||
|
|
||||||
mac.Write(salt)
|
|
||||||
return mac.Sum(nil), salt, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// MatchObjectEncryption check if encryption params are valid for provided object.
|
|
||||||
func (p EncryptionParams) MatchObjectEncryption(encInfo data.EncryptionInfo) error {
|
|
||||||
if p.Enabled() != encInfo.Enabled {
|
|
||||||
return errorsStd.New("invalid encryption view")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !encInfo.Enabled {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
hmacSalt, err := hex.DecodeString(encInfo.HMACSalt)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("invalid hmacSalt '%s': %w", encInfo.HMACSalt, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
hmacKey, err := hex.DecodeString(encInfo.HMACKey)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("invalid hmacKey '%s': %w", encInfo.HMACKey, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
mac := hmac.New(sha256.New, p.Key())
|
|
||||||
mac.Write(hmacSalt)
|
|
||||||
expectedHmacKey := mac.Sum(nil)
|
|
||||||
if !bytes.Equal(expectedHmacKey, hmacKey) {
|
|
||||||
return errorsStd.New("mismatched hmac key")
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DefaultCachesConfigs returns filled configs.
|
// DefaultCachesConfigs returns filled configs.
|
||||||
func DefaultCachesConfigs(logger *zap.Logger) *CachesConfig {
|
func DefaultCachesConfigs(logger *zap.Logger) *CachesConfig {
|
||||||
return &CachesConfig{
|
return &CachesConfig{
|
||||||
|
@ -473,253 +395,6 @@ func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||||
return n.containerList(ctx)
|
return n.containerList(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
func formEncryptedParts(header string) ([]EncryptedPart, error) {
|
|
||||||
partInfos := strings.Split(header, ",")
|
|
||||||
result := make([]EncryptedPart, len(partInfos))
|
|
||||||
|
|
||||||
for i, partInfo := range partInfos {
|
|
||||||
part, err := parseCompletedPartHeader(partInfo)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
encPartSize, err := sio.EncryptedSize(uint64(part.Size))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("compute encrypted size: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result[i] = EncryptedPart{
|
|
||||||
Part: *part,
|
|
||||||
EncryptedSize: int64(encPartSize),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type decrypter struct {
|
|
||||||
reader io.Reader
|
|
||||||
decReader io.Reader
|
|
||||||
parts []EncryptedPart
|
|
||||||
currentPart int
|
|
||||||
encryption EncryptionParams
|
|
||||||
|
|
||||||
rangeParam *RangeParams
|
|
||||||
|
|
||||||
partDataRemain int64
|
|
||||||
encPartRangeLen int64
|
|
||||||
|
|
||||||
seqNumber uint64
|
|
||||||
decLen int64
|
|
||||||
skipLen uint64
|
|
||||||
|
|
||||||
ln uint64
|
|
||||||
off uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d decrypter) decLength() int64 {
|
|
||||||
return d.decLen
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d decrypter) encLength() uint64 {
|
|
||||||
return d.ln
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d decrypter) encOffset() uint64 {
|
|
||||||
return d.off
|
|
||||||
}
|
|
||||||
|
|
||||||
func getDecryptReader(p *GetObjectParams) (*decrypter, error) {
|
|
||||||
if !p.Encryption.Enabled() {
|
|
||||||
return nil, errorsStd.New("couldn't create decrypter with disabled encryption")
|
|
||||||
}
|
|
||||||
|
|
||||||
rangeParam := p.Range
|
|
||||||
|
|
||||||
var err error
|
|
||||||
var parts []EncryptedPart
|
|
||||||
header := p.ObjectInfo.Headers[UploadCompletedParts]
|
|
||||||
if len(header) != 0 {
|
|
||||||
parts, err = formEncryptedParts(header)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("form parts: %w", err)
|
|
||||||
}
|
|
||||||
if rangeParam == nil {
|
|
||||||
decSizeHeader := p.ObjectInfo.Headers[AttributeDecryptedSize]
|
|
||||||
size, err := strconv.ParseUint(decSizeHeader, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("parse dec size header '%s': %w", decSizeHeader, err)
|
|
||||||
}
|
|
||||||
rangeParam = &RangeParams{
|
|
||||||
Start: 0,
|
|
||||||
End: size - 1,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
decSize, err := sio.DecryptedSize(uint64(p.ObjectInfo.Size))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("compute decrypted size: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
parts = []EncryptedPart{{
|
|
||||||
Part: Part{Size: int64(decSize)},
|
|
||||||
EncryptedSize: p.ObjectInfo.Size,
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
if rangeParam != nil && rangeParam.Start > rangeParam.End {
|
|
||||||
return nil, fmt.Errorf("invalid range: %d %d", rangeParam.Start, rangeParam.End)
|
|
||||||
}
|
|
||||||
|
|
||||||
decReader := &decrypter{
|
|
||||||
parts: parts,
|
|
||||||
rangeParam: rangeParam,
|
|
||||||
encryption: p.Encryption,
|
|
||||||
}
|
|
||||||
|
|
||||||
decReader.initRangeParams()
|
|
||||||
|
|
||||||
return decReader, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
blockSize = 1 << 16 // 64KB
|
|
||||||
fullBlockSize = blockSize + 32
|
|
||||||
)
|
|
||||||
|
|
||||||
func (d *decrypter) initRangeParams() {
|
|
||||||
d.partDataRemain = d.parts[d.currentPart].Size
|
|
||||||
d.encPartRangeLen = d.parts[d.currentPart].EncryptedSize
|
|
||||||
if d.rangeParam == nil {
|
|
||||||
d.decLen = d.partDataRemain
|
|
||||||
d.ln = uint64(d.encPartRangeLen)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
start, end := d.rangeParam.Start, d.rangeParam.End
|
|
||||||
|
|
||||||
var sum, encSum uint64
|
|
||||||
var partStart int
|
|
||||||
for i, part := range d.parts {
|
|
||||||
if start < sum+uint64(part.Size) {
|
|
||||||
partStart = i
|
|
||||||
break
|
|
||||||
}
|
|
||||||
sum += uint64(part.Size)
|
|
||||||
encSum += uint64(part.EncryptedSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
d.skipLen = (start - sum) % blockSize
|
|
||||||
d.seqNumber = (start - sum) / blockSize
|
|
||||||
encOffPart := d.seqNumber * fullBlockSize
|
|
||||||
d.off = encSum + encOffPart
|
|
||||||
d.encPartRangeLen = d.encPartRangeLen - int64(encOffPart)
|
|
||||||
d.partDataRemain = d.partDataRemain + int64(sum-start)
|
|
||||||
|
|
||||||
var partEnd int
|
|
||||||
for i, part := range d.parts[partStart:] {
|
|
||||||
index := partStart + i
|
|
||||||
if end < sum+uint64(part.Size) {
|
|
||||||
partEnd = index
|
|
||||||
break
|
|
||||||
}
|
|
||||||
sum += uint64(part.Size)
|
|
||||||
encSum += uint64(part.EncryptedSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
payloadPartEnd := (end - sum) / blockSize
|
|
||||||
endEnc := encSum + (payloadPartEnd+1)*fullBlockSize
|
|
||||||
|
|
||||||
endPartEnc := encSum + uint64(d.parts[partEnd].EncryptedSize)
|
|
||||||
if endPartEnc < endEnc {
|
|
||||||
endEnc = endPartEnc
|
|
||||||
}
|
|
||||||
d.ln = endEnc - d.off
|
|
||||||
d.decLen = int64(end - start + 1)
|
|
||||||
|
|
||||||
if int64(d.ln) < d.encPartRangeLen {
|
|
||||||
d.encPartRangeLen = int64(d.ln)
|
|
||||||
}
|
|
||||||
if d.decLen < d.partDataRemain {
|
|
||||||
d.partDataRemain = d.decLen
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *decrypter) updateRangeParams() {
|
|
||||||
d.partDataRemain = d.parts[d.currentPart].Size
|
|
||||||
d.encPartRangeLen = d.parts[d.currentPart].EncryptedSize
|
|
||||||
d.seqNumber = 0
|
|
||||||
d.skipLen = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *decrypter) Read(p []byte) (int, error) {
|
|
||||||
if int64(len(p)) < d.partDataRemain {
|
|
||||||
n, err := d.decReader.Read(p)
|
|
||||||
if err != nil {
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
d.partDataRemain -= int64(n)
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
n1, err := io.ReadFull(d.decReader, p[:d.partDataRemain])
|
|
||||||
if err != nil {
|
|
||||||
return n1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
d.currentPart++
|
|
||||||
if d.currentPart == len(d.parts) {
|
|
||||||
return n1, io.EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
d.updateRangeParams()
|
|
||||||
|
|
||||||
err = d.initNextDecReader()
|
|
||||||
if err != nil {
|
|
||||||
return n1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
n2, err := d.decReader.Read(p[n1:])
|
|
||||||
if err != nil {
|
|
||||||
return n1 + n2, err
|
|
||||||
}
|
|
||||||
|
|
||||||
d.partDataRemain -= int64(n2)
|
|
||||||
|
|
||||||
return n1 + n2, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *decrypter) SetReader(r io.Reader) error {
|
|
||||||
d.reader = r
|
|
||||||
return d.initNextDecReader()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *decrypter) initNextDecReader() error {
|
|
||||||
if d.reader == nil {
|
|
||||||
return errorsStd.New("reader isn't set")
|
|
||||||
}
|
|
||||||
|
|
||||||
r, err := sio.DecryptReader(io.LimitReader(d.reader, d.encPartRangeLen),
|
|
||||||
sio.Config{
|
|
||||||
MinVersion: sio.Version20,
|
|
||||||
SequenceNumber: uint32(d.seqNumber),
|
|
||||||
Key: d.encryption.Key(),
|
|
||||||
CipherSuites: []byte{sio.AES_256_GCM},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("couldn't create decrypter: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if d.skipLen > 0 {
|
|
||||||
if _, err = io.CopyN(io.Discard, r, int64(d.skipLen)); err != nil {
|
|
||||||
return fmt.Errorf("couldn't skip some bytes: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
d.decReader = r
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetObject from storage.
|
// GetObject from storage.
|
||||||
func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
|
func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
|
||||||
var params getParams
|
var params getParams
|
||||||
|
@ -727,15 +402,15 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
|
||||||
params.oid = p.ObjectInfo.ID
|
params.oid = p.ObjectInfo.ID
|
||||||
params.bktInfo = p.BucketInfo
|
params.bktInfo = p.BucketInfo
|
||||||
|
|
||||||
var decReader *decrypter
|
var decReader *encryption.Decrypter
|
||||||
if p.Encryption.Enabled() {
|
if p.Encryption.Enabled() {
|
||||||
var err error
|
var err error
|
||||||
decReader, err = getDecryptReader(p)
|
decReader, err = getDecrypter(p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("creating decrypter: %w", err)
|
return fmt.Errorf("creating decrypter: %w", err)
|
||||||
}
|
}
|
||||||
params.off = decReader.encOffset()
|
params.off = decReader.EncryptedOffset()
|
||||||
params.ln = decReader.encLength()
|
params.ln = decReader.EncryptedLength()
|
||||||
} else {
|
} else {
|
||||||
if p.Range != nil {
|
if p.Range != nil {
|
||||||
if p.Range.Start > p.Range.End {
|
if p.Range.Start > p.Range.End {
|
||||||
|
@ -764,18 +439,47 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
|
||||||
if err = decReader.SetReader(payload); err != nil {
|
if err = decReader.SetReader(payload); err != nil {
|
||||||
return fmt.Errorf("set reader to decrypter: %w", err)
|
return fmt.Errorf("set reader to decrypter: %w", err)
|
||||||
}
|
}
|
||||||
r = io.LimitReader(decReader, decReader.decLength())
|
r = io.LimitReader(decReader, int64(decReader.DecryptedLength()))
|
||||||
}
|
}
|
||||||
|
|
||||||
// copy full payload
|
// copy full payload
|
||||||
written, err := io.CopyBuffer(p.Writer, r, buf)
|
written, err := io.CopyBuffer(p.Writer, r, buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("copy object payload written: '%d', decLength: '%d', params.ln: '%d' : %w", written, decReader.decLength(), params.ln, err)
|
return fmt.Errorf("copy object payload written: '%d', decLength: '%d', params.ln: '%d' : %w", written, decReader.DecryptedLength(), params.ln, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getDecrypter(p *GetObjectParams) (*encryption.Decrypter, error) {
|
||||||
|
var encRange *encryption.Range
|
||||||
|
if p.Range != nil {
|
||||||
|
encRange = &encryption.Range{Start: p.Range.Start, End: p.Range.End}
|
||||||
|
}
|
||||||
|
|
||||||
|
header := p.ObjectInfo.Headers[UploadCompletedParts]
|
||||||
|
if len(header) == 0 {
|
||||||
|
return encryption.NewDecrypter(p.Encryption, uint64(p.ObjectInfo.Size), encRange)
|
||||||
|
}
|
||||||
|
|
||||||
|
decryptedObjectSize, err := strconv.ParseUint(p.ObjectInfo.Headers[AttributeDecryptedSize], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse decrypted size: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
splits := strings.Split(header, ",")
|
||||||
|
sizes := make([]uint64, len(splits))
|
||||||
|
for i, splitInfo := range splits {
|
||||||
|
part, err := ParseCompletedPartHeader(splitInfo)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse completed part: %w", err)
|
||||||
|
}
|
||||||
|
sizes[i] = uint64(part.Size)
|
||||||
|
}
|
||||||
|
|
||||||
|
return encryption.NewMultipartDecrypter(p.Encryption, decryptedObjectSize, sizes, encRange)
|
||||||
|
}
|
||||||
|
|
||||||
// GetObjectInfo returns meta information about the object.
|
// GetObjectInfo returns meta information about the object.
|
||||||
func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) {
|
func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) {
|
||||||
extendedObjectInfo, err := n.GetExtendedObjectInfo(ctx, p)
|
extendedObjectInfo, err := n.GetExtendedObjectInfo(ctx, p)
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
"github.com/minio/sio"
|
"github.com/minio/sio"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -40,7 +41,7 @@ type (
|
||||||
UploadID string
|
UploadID string
|
||||||
Bkt *data.BucketInfo
|
Bkt *data.BucketInfo
|
||||||
Key string
|
Key string
|
||||||
Encryption EncryptionParams
|
Encryption encryption.Params
|
||||||
}
|
}
|
||||||
|
|
||||||
CreateMultipartParams struct {
|
CreateMultipartParams struct {
|
||||||
|
@ -190,7 +191,7 @@ func (n *layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, er
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
|
func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
|
||||||
encInfo := formEncryptionInfo(multipartInfo.Meta)
|
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||||
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||||
n.log.Warn("mismatched obj encryptionInfo", zap.Error(err))
|
n.log.Warn("mismatched obj encryptionInfo", zap.Error(err))
|
||||||
return nil, errors.GetAPIError(errors.ErrInvalidEncryptionParameters)
|
return nil, errors.GetAPIError(errors.ErrInvalidEncryptionParameters)
|
||||||
|
@ -356,7 +357,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
encInfo := formEncryptionInfo(multipartInfo.Meta)
|
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||||
|
|
||||||
if len(partsInfo) < len(p.Parts) {
|
if len(partsInfo) < len(p.Parts) {
|
||||||
return nil, nil, errors.GetAPIError(errors.ErrInvalidPart)
|
return nil, nil, errors.GetAPIError(errors.ErrInvalidPart)
|
||||||
|
@ -545,7 +546,7 @@ func (n *layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
encInfo := formEncryptionInfo(multipartInfo.Meta)
|
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||||
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||||
n.log.Warn("mismatched obj encryptionInfo", zap.Error(err))
|
n.log.Warn("mismatched obj encryptionInfo", zap.Error(err))
|
||||||
return nil, errors.GetAPIError(errors.ErrInvalidEncryptionParameters)
|
return nil, errors.GetAPIError(errors.ErrInvalidEncryptionParameters)
|
||||||
|
|
|
@ -161,7 +161,7 @@ func encryptionReader(r io.Reader, size uint64, key []byte) (io.Reader, uint64,
|
||||||
return r, encSize, nil
|
return r, encSize, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseCompletedPartHeader(hdr string) (*Part, error) {
|
func ParseCompletedPartHeader(hdr string) (*Part, error) {
|
||||||
// partInfo[0] -- part number, partInfo[1] -- part size, partInfo[2] -- checksum
|
// partInfo[0] -- part number, partInfo[1] -- part size, partInfo[2] -- checksum
|
||||||
partInfo := strings.Split(hdr, "-")
|
partInfo := strings.Split(hdr, "-")
|
||||||
if len(partInfo) != 3 {
|
if len(partInfo) != 3 {
|
||||||
|
@ -268,9 +268,8 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object
|
||||||
n.listsCache.CleanCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
|
n.listsCache.CleanCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
|
||||||
|
|
||||||
objInfo := &data.ObjectInfo{
|
objInfo := &data.ObjectInfo{
|
||||||
ID: id,
|
ID: id,
|
||||||
CID: p.BktInfo.CID,
|
CID: p.BktInfo.CID,
|
||||||
EncryptionInfo: formEncryptionInfo(p.Header),
|
|
||||||
|
|
||||||
Owner: own,
|
Owner: own,
|
||||||
Bucket: p.BktInfo.Name,
|
Bucket: p.BktInfo.Name,
|
||||||
|
|
|
@ -9,6 +9,8 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
|
@ -83,10 +85,9 @@ func objectInfoFromMeta(bkt *data.BucketInfo, meta *object.Object) *data.ObjectI
|
||||||
objID, _ := meta.ID()
|
objID, _ := meta.ID()
|
||||||
payloadChecksum, _ := meta.PayloadChecksum()
|
payloadChecksum, _ := meta.PayloadChecksum()
|
||||||
return &data.ObjectInfo{
|
return &data.ObjectInfo{
|
||||||
ID: objID,
|
ID: objID,
|
||||||
CID: bkt.CID,
|
CID: bkt.CID,
|
||||||
IsDir: false,
|
IsDir: false,
|
||||||
EncryptionInfo: formEncryptionInfo(headers),
|
|
||||||
|
|
||||||
Bucket: bkt.Name,
|
Bucket: bkt.Name,
|
||||||
Name: filenameFromObject(meta),
|
Name: filenameFromObject(meta),
|
||||||
|
@ -99,9 +100,9 @@ func objectInfoFromMeta(bkt *data.BucketInfo, meta *object.Object) *data.ObjectI
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func formEncryptionInfo(headers map[string]string) data.EncryptionInfo {
|
func FormEncryptionInfo(headers map[string]string) encryption.ObjectEncryption {
|
||||||
algorithm := headers[AttributeEncryptionAlgorithm]
|
algorithm := headers[AttributeEncryptionAlgorithm]
|
||||||
return data.EncryptionInfo{
|
return encryption.ObjectEncryption{
|
||||||
Enabled: len(algorithm) > 0,
|
Enabled: len(algorithm) > 0,
|
||||||
Algorithm: algorithm,
|
Algorithm: algorithm,
|
||||||
HMACKey: headers[AttributeHMACKey],
|
HMACKey: headers[AttributeHMACKey],
|
||||||
|
@ -109,7 +110,7 @@ func formEncryptionInfo(headers map[string]string) data.EncryptionInfo {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func addEncryptionHeaders(meta map[string]string, enc EncryptionParams) error {
|
func addEncryptionHeaders(meta map[string]string, enc encryption.Params) error {
|
||||||
meta[AttributeEncryptionAlgorithm] = AESEncryptionAlgorithm
|
meta[AttributeEncryptionAlgorithm] = AESEncryptionAlgorithm
|
||||||
hmacKey, hmacSalt, err := enc.HMAC()
|
hmacKey, hmacSalt, err := enc.HMAC()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
Loading…
Reference in a new issue