Add chunk size flag to frostfs-cli object put #491

Closed
dstepanov-yadro wants to merge 1 commit from dstepanov-yadro/frostfs-node:feat/object-put-chunk-size into master
3 changed files with 25 additions and 3 deletions

View file

@ -338,6 +338,7 @@ type PutObjectPrm struct {
headerCallback func(*object.Object) headerCallback func(*object.Object)
prepareLocally bool prepareLocally bool
chunkSize int
} }
// SetHeader sets object header. // SetHeader sets object header.
@ -368,6 +369,11 @@ func (x *PutObjectPrm) PrepareLocally() {
x.prepareLocally = true x.prepareLocally = true
} }
// SetGRPCChunkSize sets the maximum size of single gRPC message.
func (x *PutObjectPrm) SetGRPCChunkSize(v int) {
x.chunkSize = v
}
func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) { func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) {
var putPrm client.PrmObjectPutInit var putPrm client.PrmObjectPutInit
if !x.prepareLocally && x.sessionToken != nil { if !x.prepareLocally && x.sessionToken != nil {
@ -384,6 +390,9 @@ func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPut
putPrm.WithXHeaders(x.xHeaders...) putPrm.WithXHeaders(x.xHeaders...)
putPrm.SetCopiesNumberByVectors(x.copyNum) putPrm.SetCopiesNumberByVectors(x.copyNum)
if x.chunkSize > 0 {
putPrm.SetGRPCPayloadChunkLen(x.chunkSize)
}
if x.prepareLocally { if x.prepareLocally {
res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{}) res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{})

View file

@ -14,6 +14,7 @@ import (
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -27,6 +28,7 @@ const (
notificationFlag = "notify" notificationFlag = "notify"
copiesNumberFlag = "copies-number" copiesNumberFlag = "copies-number"
prepareLocallyFlag = "prepare-locally" prepareLocallyFlag = "prepare-locally"
chunkSizeFlag = "chunk-size"
) )
var putExpiredOn uint64 var putExpiredOn uint64
@ -61,6 +63,7 @@ func initObjectPutCmd() {
flags.Bool(binaryFlag, false, "Deserialize object structure from given file.") flags.Bool(binaryFlag, false, "Deserialize object structure from given file.")
flags.String(copiesNumberFlag, "", "Number of copies of the object to store within the RPC call") flags.String(copiesNumberFlag, "", "Number of copies of the object to store within the RPC call")
flags.String(chunkSizeFlag, "", "The maximum size of payload chunk sended to FrostFS within single request")
} }
func putObject(cmd *cobra.Command, _ []string) { func putObject(cmd *cobra.Command, _ []string) {
@ -129,6 +132,7 @@ func putObject(cmd *cobra.Command, _ []string) {
copyNum, err := cmd.Flags().GetString(copiesNumberFlag) copyNum, err := cmd.Flags().GetString(copiesNumberFlag)
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum)) prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum))
prm.SetGRPCChunkSize(parseChunkSize(cmd))
res, err := internalclient.PutObject(cmd.Context(), prm) res, err := internalclient.PutObject(cmd.Context(), prm)
if p != nil { if p != nil {
@ -286,3 +290,12 @@ func parseObjectNotifications(cmd *cobra.Command) (*object.NotificationInfo, err
return ni, nil return ni, nil
} }
func parseChunkSize(cmd *cobra.Command) int {
raw, err := cmd.Flags().GetString(chunkSizeFlag)
commonCmd.ExitOnErr(cmd, "failed to get chunk size flag value: %w", err)
if raw == "" {
return 0
}
return int(config.ParseSizeInBytes(raw))
}

View file

@ -158,7 +158,7 @@ func IntSafe(c *Config, name string) int64 {
// Returns 0 if a value can't be casted. // Returns 0 if a value can't be casted.
func SizeInBytesSafe(c *Config, name string) uint64 { func SizeInBytesSafe(c *Config, name string) uint64 {
s := StringSafe(c, name) s := StringSafe(c, name)
return parseSizeInBytes(s) return ParseSizeInBytes(s)
} }
// The following code is taken from https://github.com/spf13/viper/blob/master/util.go // The following code is taken from https://github.com/spf13/viper/blob/master/util.go
@ -189,8 +189,8 @@ func safeMul(size float64, multiplier uint64) uint64 {
return lo return lo
} }
// parseSizeInBytes converts strings like 1GB or 12 mb into an unsigned integer number of bytes. // ParseSizeInBytes converts strings like 1GB or 12 mb into an unsigned integer number of bytes.
func parseSizeInBytes(sizeStr string) uint64 { func ParseSizeInBytes(sizeStr string) uint64 {
sizeStr = strings.TrimSpace(sizeStr) sizeStr = strings.TrimSpace(sizeStr)
lastChar := len(sizeStr) - 1 lastChar := len(sizeStr) - 1
multiplier := uint64(1) multiplier := uint64(1)