Add chunk size flag to frostfs-cli object put #491

Closed
dstepanov-yadro wants to merge 1 commit from dstepanov-yadro/frostfs-node:feat/object-put-chunk-size into master
3 changed files with 25 additions and 3 deletions

View file

@ -338,6 +338,7 @@ type PutObjectPrm struct {
headerCallback func(*object.Object)
prepareLocally bool
chunkSize int
}
// SetHeader sets object header.
@ -368,6 +369,11 @@ func (x *PutObjectPrm) PrepareLocally() {
x.prepareLocally = true
}
// SetGRPCChunkSize sets the maximum size of single gRPC message.
func (x *PutObjectPrm) SetGRPCChunkSize(v int) {
x.chunkSize = v
}
func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) {
var putPrm client.PrmObjectPutInit
if !x.prepareLocally && x.sessionToken != nil {
@ -384,6 +390,9 @@ func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPut
putPrm.WithXHeaders(x.xHeaders...)
putPrm.SetCopiesNumberByVectors(x.copyNum)
if x.chunkSize > 0 {
putPrm.SetGRPCPayloadChunkLen(x.chunkSize)
}
if x.prepareLocally {
res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{})

View file

@ -14,6 +14,7 @@ import (
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -27,6 +28,7 @@ const (
notificationFlag = "notify"
copiesNumberFlag = "copies-number"
prepareLocallyFlag = "prepare-locally"
chunkSizeFlag = "chunk-size"
)
var putExpiredOn uint64
@ -61,6 +63,7 @@ func initObjectPutCmd() {
flags.Bool(binaryFlag, false, "Deserialize object structure from given file.")
flags.String(copiesNumberFlag, "", "Number of copies of the object to store within the RPC call")
flags.String(chunkSizeFlag, "", "The maximum size of payload chunk sended to FrostFS within single request")
}
func putObject(cmd *cobra.Command, _ []string) {
@ -129,6 +132,7 @@ func putObject(cmd *cobra.Command, _ []string) {
copyNum, err := cmd.Flags().GetString(copiesNumberFlag)
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum))
prm.SetGRPCChunkSize(parseChunkSize(cmd))
res, err := internalclient.PutObject(cmd.Context(), prm)
if p != nil {
@ -286,3 +290,12 @@ func parseObjectNotifications(cmd *cobra.Command) (*object.NotificationInfo, err
return ni, nil
}
func parseChunkSize(cmd *cobra.Command) int {
raw, err := cmd.Flags().GetString(chunkSizeFlag)
commonCmd.ExitOnErr(cmd, "failed to get chunk size flag value: %w", err)
if raw == "" {
return 0
}
return int(config.ParseSizeInBytes(raw))
}

View file

@ -158,7 +158,7 @@ func IntSafe(c *Config, name string) int64 {
// Returns 0 if a value can't be casted.
func SizeInBytesSafe(c *Config, name string) uint64 {
s := StringSafe(c, name)
return parseSizeInBytes(s)
return ParseSizeInBytes(s)
}
// The following code is taken from https://github.com/spf13/viper/blob/master/util.go
@ -189,8 +189,8 @@ func safeMul(size float64, multiplier uint64) uint64 {
return lo
}
// parseSizeInBytes converts strings like 1GB or 12 mb into an unsigned integer number of bytes.
func parseSizeInBytes(sizeStr string) uint64 {
// ParseSizeInBytes converts strings like 1GB or 12 mb into an unsigned integer number of bytes.
func ParseSizeInBytes(sizeStr string) uint64 {
sizeStr = strings.TrimSpace(sizeStr)
lastChar := len(sizeStr) - 1
multiplier := uint64(1)