561 lines
16 KiB
Go
561 lines
16 KiB
Go
package layer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/md5"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"mime"
|
|
"net/http"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/detector"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
"github.com/minio/sio"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type (
|
|
getParams struct {
|
|
// payload range
|
|
off, ln uint64
|
|
|
|
objInfo *data.ObjectInfo
|
|
bktInfo *data.BucketInfo
|
|
}
|
|
|
|
getFrostFSParams struct {
|
|
// payload range
|
|
off, ln uint64
|
|
|
|
oid oid.ID
|
|
bktInfo *data.BucketInfo
|
|
}
|
|
|
|
DeleteMarkerError struct {
|
|
ErrorCode apiErrors.ErrorCode
|
|
}
|
|
)
|
|
|
|
func (e DeleteMarkerError) Error() string {
|
|
return "object is delete marker"
|
|
}
|
|
|
|
const (
|
|
continuationToken = "<continuation-token>"
|
|
)
|
|
|
|
func newAddress(cnr cid.ID, obj oid.ID) oid.Address {
|
|
var addr oid.Address
|
|
addr.SetContainer(cnr)
|
|
addr.SetObject(obj)
|
|
return addr
|
|
}
|
|
|
|
// objectHead returns all object's headers.
|
|
func (n *Layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) (*object.Object, error) {
|
|
prm := PrmObjectHead{
|
|
Container: bktInfo.CID,
|
|
Object: idObj,
|
|
}
|
|
|
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
|
|
|
return n.frostFS.HeadObject(ctx, prm)
|
|
}
|
|
|
|
func (n *Layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Reader, error) {
|
|
if _, isCombined := p.objInfo.Headers[MultipartObjectSize]; !isCombined {
|
|
return n.initFrostFSObjectPayloadReader(ctx, getFrostFSParams{
|
|
off: p.off,
|
|
ln: p.ln,
|
|
oid: p.objInfo.ID,
|
|
bktInfo: p.bktInfo,
|
|
})
|
|
}
|
|
|
|
combinedObj, err := n.objectGet(ctx, p.bktInfo, p.objInfo.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("get combined object '%s': %w", p.objInfo.ID.EncodeToString(), err)
|
|
}
|
|
|
|
var parts []*data.PartInfo
|
|
if err = json.NewDecoder(combinedObj.Payload).Decode(&parts); err != nil {
|
|
return nil, fmt.Errorf("unmarshal combined object parts: %w", err)
|
|
}
|
|
|
|
isEncrypted := FormEncryptionInfo(p.objInfo.Headers).Enabled
|
|
objParts := make([]partObj, len(parts))
|
|
for i, part := range parts {
|
|
size := part.Size
|
|
if isEncrypted {
|
|
if size, err = sio.EncryptedSize(part.Size); err != nil {
|
|
return nil, fmt.Errorf("compute encrypted size: %w", err)
|
|
}
|
|
}
|
|
|
|
objParts[i] = partObj{
|
|
OID: part.OID,
|
|
Size: size,
|
|
}
|
|
}
|
|
|
|
return newMultiObjectReader(ctx, multiObjectReaderConfig{
|
|
layer: n,
|
|
off: p.off,
|
|
ln: p.ln,
|
|
parts: objParts,
|
|
bktInfo: p.bktInfo,
|
|
})
|
|
}
|
|
|
|
// initializes payload reader of the FrostFS object.
|
|
// Zero range corresponds to full payload (panics if only offset is set).
|
|
func (n *Layer) initFrostFSObjectPayloadReader(ctx context.Context, p getFrostFSParams) (io.Reader, error) {
|
|
var prmAuth PrmAuth
|
|
n.prepareAuthParameters(ctx, &prmAuth, p.bktInfo.Owner)
|
|
|
|
if p.off+p.ln != 0 {
|
|
prm := PrmObjectRange{
|
|
PrmAuth: prmAuth,
|
|
Container: p.bktInfo.CID,
|
|
Object: p.oid,
|
|
PayloadRange: [2]uint64{p.off, p.ln},
|
|
}
|
|
|
|
return n.frostFS.RangeObject(ctx, prm)
|
|
}
|
|
|
|
prm := PrmObjectGet{
|
|
PrmAuth: prmAuth,
|
|
Container: p.bktInfo.CID,
|
|
Object: p.oid,
|
|
}
|
|
|
|
res, err := n.frostFS.GetObject(ctx, prm)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return res.Payload, nil
|
|
}
|
|
|
|
// objectGet returns an object with payload in the object.
|
|
func (n *Layer) objectGet(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (*Object, error) {
|
|
return n.objectGetBase(ctx, bktInfo, objID, PrmAuth{})
|
|
}
|
|
|
|
// objectGetWithAuth returns an object with payload in the object. Uses provided PrmAuth.
|
|
func (n *Layer) objectGetWithAuth(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, auth PrmAuth) (*Object, error) {
|
|
return n.objectGetBase(ctx, bktInfo, objID, auth)
|
|
}
|
|
|
|
func (n *Layer) objectGetBase(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, auth PrmAuth) (*Object, error) {
|
|
prm := PrmObjectGet{
|
|
PrmAuth: auth,
|
|
Container: bktInfo.CID,
|
|
Object: objID,
|
|
}
|
|
|
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
|
|
|
return n.frostFS.GetObject(ctx, prm)
|
|
}
|
|
|
|
// MimeByFilePath detect mime type by file path extension.
|
|
func MimeByFilePath(path string) string {
|
|
ext := filepath.Ext(path)
|
|
if len(ext) == 0 {
|
|
return ""
|
|
}
|
|
return mime.TypeByExtension(ext)
|
|
}
|
|
|
|
func encryptionReader(r io.Reader, size uint64, key []byte) (io.Reader, uint64, error) {
|
|
encSize, err := sio.EncryptedSize(size)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("failed to compute enc size: %w", err)
|
|
}
|
|
|
|
r, err = sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, MaxVersion: sio.Version20, Key: key, CipherSuites: []byte{sio.AES_256_GCM}})
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("couldn't create encrypter: %w", err)
|
|
}
|
|
|
|
return r, encSize, nil
|
|
}
|
|
|
|
func ParseCompletedPartHeader(hdr string) (*Part, error) {
|
|
// partInfo[0] -- part number, partInfo[1] -- part size, partInfo[2] -- checksum
|
|
partInfo := strings.Split(hdr, "-")
|
|
if len(partInfo) != 3 {
|
|
return nil, fmt.Errorf("invalid completed part header")
|
|
}
|
|
num, err := strconv.Atoi(partInfo[0])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid completed part number '%s': %w", partInfo[0], err)
|
|
}
|
|
size, err := strconv.ParseUint(partInfo[1], 10, 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid completed part size '%s': %w", partInfo[1], err)
|
|
}
|
|
|
|
return &Part{
|
|
ETag: partInfo[2],
|
|
PartNumber: num,
|
|
Size: size,
|
|
}, nil
|
|
}
|
|
|
|
// PutObject stores object into FrostFS, took payload from io.Reader.
|
|
func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) {
|
|
bktSettings, err := n.GetBucketSettings(ctx, p.BktInfo)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't get versioning settings object: %w", err)
|
|
}
|
|
|
|
r := p.Reader
|
|
if p.Encryption.Enabled() {
|
|
p.Header[AttributeDecryptedSize] = strconv.FormatUint(p.Size, 10)
|
|
if err = addEncryptionHeaders(p.Header, p.Encryption); err != nil {
|
|
return nil, fmt.Errorf("add encryption header: %w", err)
|
|
}
|
|
|
|
var encSize uint64
|
|
if r, encSize, err = encryptionReader(p.Reader, p.Size, p.Encryption.Key()); err != nil {
|
|
return nil, fmt.Errorf("create encrypter: %w", err)
|
|
}
|
|
p.Size = encSize
|
|
}
|
|
|
|
if r != nil {
|
|
if len(p.Header[api.ContentType]) == 0 {
|
|
if contentType := MimeByFilePath(p.Object); len(contentType) == 0 {
|
|
d := detector.NewDetector(r, http.DetectContentType)
|
|
if contentType, err := d.Detect(); err == nil {
|
|
p.Header[api.ContentType] = contentType
|
|
}
|
|
r = d.RestoredReader()
|
|
} else {
|
|
p.Header[api.ContentType] = contentType
|
|
}
|
|
}
|
|
}
|
|
|
|
prm := PrmObjectCreate{
|
|
Container: p.BktInfo.CID,
|
|
PayloadSize: p.Size,
|
|
Filepath: p.Object,
|
|
Payload: r,
|
|
CreationTime: TimeNow(ctx),
|
|
CopiesNumber: p.CopiesNumbers,
|
|
}
|
|
|
|
prm.Attributes = make([][2]string, 0, len(p.Header))
|
|
|
|
for k, v := range p.Header {
|
|
prm.Attributes = append(prm.Attributes, [2]string{k, v})
|
|
}
|
|
|
|
createdObj, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(p.ContentMD5) > 0 {
|
|
headerMd5Hash, err := base64.StdEncoding.DecodeString(p.ContentMD5)
|
|
if err != nil {
|
|
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
|
|
}
|
|
if !bytes.Equal(headerMd5Hash, createdObj.MD5Sum) {
|
|
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
|
if err != nil {
|
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
|
}
|
|
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
|
|
}
|
|
}
|
|
|
|
if !p.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
|
|
contentHashBytes, err := hex.DecodeString(p.ContentSHA256Hash)
|
|
if err != nil {
|
|
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
|
}
|
|
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
|
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
|
if err != nil {
|
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
|
}
|
|
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
|
}
|
|
}
|
|
|
|
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
|
now := TimeNow(ctx)
|
|
newVersion := &data.NodeVersion{
|
|
BaseNodeVersion: data.BaseNodeVersion{
|
|
OID: createdObj.ID,
|
|
ETag: hex.EncodeToString(createdObj.HashSum),
|
|
FilePath: p.Object,
|
|
Size: p.Size,
|
|
Created: &now,
|
|
Owner: &n.gateOwner,
|
|
CreationEpoch: createdObj.CreationEpoch,
|
|
},
|
|
IsUnversioned: !bktSettings.VersioningEnabled(),
|
|
IsCombined: p.Header[MultipartObjectSize] != "",
|
|
}
|
|
if len(p.CompleteMD5Hash) > 0 {
|
|
newVersion.MD5 = p.CompleteMD5Hash
|
|
} else {
|
|
newVersion.MD5 = hex.EncodeToString(createdObj.MD5Sum)
|
|
}
|
|
|
|
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
|
return nil, fmt.Errorf("couldn't add new verion to tree service: %w", err)
|
|
}
|
|
|
|
if p.Lock != nil && (p.Lock.Retention != nil || p.Lock.LegalHold != nil) {
|
|
putLockInfoPrms := &PutLockInfoParams{
|
|
ObjVersion: &data.ObjectVersion{
|
|
BktInfo: p.BktInfo,
|
|
ObjectName: p.Object,
|
|
VersionID: createdObj.ID.EncodeToString(),
|
|
},
|
|
NewLock: p.Lock,
|
|
CopiesNumbers: p.CopiesNumbers,
|
|
NodeVersion: newVersion, // provide new version to make one less tree service call in PutLockInfo
|
|
}
|
|
|
|
if err = n.PutLockInfo(ctx, putLockInfoPrms); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
n.cache.CleanListCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
|
|
|
|
objInfo := &data.ObjectInfo{
|
|
ID: createdObj.ID,
|
|
CID: p.BktInfo.CID,
|
|
|
|
Owner: n.gateOwner,
|
|
Bucket: p.BktInfo.Name,
|
|
Name: p.Object,
|
|
Size: createdObj.Size,
|
|
Created: prm.CreationTime,
|
|
Headers: p.Header,
|
|
ContentType: p.Header[api.ContentType],
|
|
HashSum: newVersion.ETag,
|
|
MD5Sum: newVersion.MD5,
|
|
}
|
|
|
|
extendedObjInfo := &data.ExtendedObjectInfo{
|
|
ObjectInfo: objInfo,
|
|
NodeVersion: newVersion,
|
|
}
|
|
|
|
n.cache.PutObjectWithName(n.BearerOwner(ctx), extendedObjInfo)
|
|
|
|
return extendedObjInfo, nil
|
|
}
|
|
|
|
func (n *Layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ExtendedObjectInfo, error) {
|
|
owner := n.BearerOwner(ctx)
|
|
if extObjInfo := n.cache.GetLastObject(owner, bkt.Name, objectName); extObjInfo != nil {
|
|
return extObjInfo, nil
|
|
}
|
|
|
|
node, err := n.treeService.GetLatestVersion(ctx, bkt, objectName)
|
|
if err != nil {
|
|
if errors.Is(err, ErrNodeNotFound) {
|
|
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchKey), err.Error())
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
if node.IsDeleteMarker {
|
|
return nil, DeleteMarkerError{ErrorCode: apiErrors.ErrNoSuchKey}
|
|
}
|
|
|
|
meta, err := n.objectHead(ctx, bkt, node.OID)
|
|
if err != nil {
|
|
if client.IsErrObjectNotFound(err) {
|
|
return nil, fmt.Errorf("%w: %s; %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchKey), err.Error(), node.OID.EncodeToString())
|
|
}
|
|
return nil, err
|
|
}
|
|
objInfo := objectInfoFromMeta(bkt, meta)
|
|
objInfo.MD5Sum = node.MD5
|
|
|
|
extObjInfo := &data.ExtendedObjectInfo{
|
|
ObjectInfo: objInfo,
|
|
NodeVersion: node,
|
|
}
|
|
|
|
n.cache.PutObjectWithName(owner, extObjInfo)
|
|
|
|
return extObjInfo, nil
|
|
}
|
|
|
|
func (n *Layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
|
|
var err error
|
|
var foundVersion *data.NodeVersion
|
|
if p.VersionID == data.UnversionedObjectVersionID {
|
|
foundVersion, err = n.treeService.GetUnversioned(ctx, bkt, p.Object)
|
|
if err != nil {
|
|
if errors.Is(err, ErrNodeNotFound) {
|
|
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion), err.Error())
|
|
}
|
|
return nil, err
|
|
}
|
|
} else {
|
|
versions, err := n.treeService.GetVersions(ctx, bkt, p.Object)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't get versions: %w", err)
|
|
}
|
|
|
|
for _, version := range versions {
|
|
if version.OID.EncodeToString() == p.VersionID {
|
|
foundVersion = version
|
|
break
|
|
}
|
|
}
|
|
if foundVersion == nil {
|
|
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion))
|
|
}
|
|
}
|
|
|
|
owner := n.BearerOwner(ctx)
|
|
if extObjInfo := n.cache.GetObject(owner, newAddress(bkt.CID, foundVersion.OID)); extObjInfo != nil {
|
|
return extObjInfo, nil
|
|
}
|
|
|
|
if foundVersion.IsDeleteMarker {
|
|
return nil, DeleteMarkerError{ErrorCode: apiErrors.ErrMethodNotAllowed}
|
|
}
|
|
|
|
meta, err := n.objectHead(ctx, bkt, foundVersion.OID)
|
|
if err != nil {
|
|
if client.IsErrObjectNotFound(err) {
|
|
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion), err.Error())
|
|
}
|
|
return nil, err
|
|
}
|
|
objInfo := objectInfoFromMeta(bkt, meta)
|
|
objInfo.MD5Sum = foundVersion.MD5
|
|
|
|
extObjInfo := &data.ExtendedObjectInfo{
|
|
ObjectInfo: objInfo,
|
|
NodeVersion: foundVersion,
|
|
}
|
|
|
|
n.cache.PutObject(owner, extObjInfo)
|
|
|
|
return extObjInfo, nil
|
|
}
|
|
|
|
// objectDelete puts tombstone object into frostfs.
|
|
func (n *Layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) error {
|
|
return n.objectDeleteBase(ctx, bktInfo, idObj, PrmAuth{})
|
|
}
|
|
|
|
// objectDeleteWithAuth puts tombstone object into frostfs. Uses provided PrmAuth.
|
|
func (n *Layer) objectDeleteWithAuth(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID, auth PrmAuth) error {
|
|
return n.objectDeleteBase(ctx, bktInfo, idObj, auth)
|
|
}
|
|
|
|
func (n *Layer) objectDeleteBase(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID, auth PrmAuth) error {
|
|
prm := PrmObjectDelete{
|
|
PrmAuth: auth,
|
|
Container: bktInfo.CID,
|
|
Object: idObj,
|
|
}
|
|
|
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
|
|
|
n.cache.DeleteObject(newAddress(bktInfo.CID, idObj))
|
|
|
|
return n.frostFS.DeleteObject(ctx, prm)
|
|
}
|
|
|
|
// objectPutAndHash prepare auth parameters and invoke frostfs.CreateObject.
|
|
func (n *Layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (*data.CreatedObjectInfo, error) {
|
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
|
prm.ClientCut = n.features.ClientCut()
|
|
prm.BufferMaxSize = n.features.BufferMaxSizeForPut()
|
|
prm.WithoutHomomorphicHash = bktInfo.HomomorphicHashDisabled
|
|
var size uint64
|
|
hash := sha256.New()
|
|
md5Hash := md5.New()
|
|
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
|
size += uint64(len(buf))
|
|
hash.Write(buf)
|
|
md5Hash.Write(buf)
|
|
})
|
|
res, err := n.frostFS.CreateObject(ctx, prm)
|
|
if err != nil {
|
|
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
|
|
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
|
|
}
|
|
|
|
return nil, err
|
|
}
|
|
return &data.CreatedObjectInfo{
|
|
ID: res.ObjectID,
|
|
Size: size,
|
|
HashSum: hash.Sum(nil),
|
|
MD5Sum: md5Hash.Sum(nil),
|
|
CreationEpoch: res.CreationEpoch,
|
|
}, nil
|
|
}
|
|
|
|
type logWrapper struct {
|
|
log *zap.Logger
|
|
}
|
|
|
|
func (l *logWrapper) Printf(format string, args ...interface{}) {
|
|
l.log.Info(fmt.Sprintf(format, args...))
|
|
}
|
|
|
|
func IsSystemHeader(key string) bool {
|
|
_, ok := api.SystemMetadata[key]
|
|
return ok || strings.HasPrefix(key, api.FrostFSSystemMetadataPrefix)
|
|
}
|
|
|
|
func wrapReader(input io.Reader, bufSize int, f func(buf []byte)) io.Reader {
|
|
if input == nil {
|
|
return nil
|
|
}
|
|
|
|
r, w := io.Pipe()
|
|
go func() {
|
|
var buf = make([]byte, bufSize)
|
|
for {
|
|
n, err := input.Read(buf)
|
|
if n > 0 {
|
|
f(buf[:n])
|
|
_, _ = w.Write(buf[:n]) // ignore error, input is not ReadCloser
|
|
}
|
|
if err != nil {
|
|
_ = w.CloseWithError(err)
|
|
break
|
|
}
|
|
}
|
|
}()
|
|
return r
|
|
}
|