Add chunk size flag to frostfs-cli object put #491
3 changed files with 25 additions and 3 deletions
|
@ -338,6 +338,7 @@ type PutObjectPrm struct {
|
||||||
headerCallback func(*object.Object)
|
headerCallback func(*object.Object)
|
||||||
|
|
||||||
prepareLocally bool
|
prepareLocally bool
|
||||||
|
chunkSize int
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetHeader sets object header.
|
// SetHeader sets object header.
|
||||||
|
@ -368,6 +369,11 @@ func (x *PutObjectPrm) PrepareLocally() {
|
||||||
x.prepareLocally = true
|
x.prepareLocally = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetGRPCChunkSize sets the maximum size of single gRPC message.
|
||||||
|
func (x *PutObjectPrm) SetGRPCChunkSize(v int) {
|
||||||
|
x.chunkSize = v
|
||||||
|
}
|
||||||
|
|
||||||
func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) {
|
func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) {
|
||||||
var putPrm client.PrmObjectPutInit
|
var putPrm client.PrmObjectPutInit
|
||||||
if !x.prepareLocally && x.sessionToken != nil {
|
if !x.prepareLocally && x.sessionToken != nil {
|
||||||
|
@ -384,6 +390,9 @@ func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPut
|
||||||
|
|
||||||
putPrm.WithXHeaders(x.xHeaders...)
|
putPrm.WithXHeaders(x.xHeaders...)
|
||||||
putPrm.SetCopiesNumberByVectors(x.copyNum)
|
putPrm.SetCopiesNumberByVectors(x.copyNum)
|
||||||
|
if x.chunkSize > 0 {
|
||||||
|
putPrm.SetGRPCPayloadChunkLen(x.chunkSize)
|
||||||
|
}
|
||||||
|
|
||||||
if x.prepareLocally {
|
if x.prepareLocally {
|
||||||
res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{})
|
res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{})
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
|
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
|
||||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
@ -27,6 +28,7 @@ const (
|
||||||
notificationFlag = "notify"
|
notificationFlag = "notify"
|
||||||
copiesNumberFlag = "copies-number"
|
copiesNumberFlag = "copies-number"
|
||||||
prepareLocallyFlag = "prepare-locally"
|
prepareLocallyFlag = "prepare-locally"
|
||||||
|
chunkSizeFlag = "chunk-size"
|
||||||
)
|
)
|
||||||
|
|
||||||
var putExpiredOn uint64
|
var putExpiredOn uint64
|
||||||
|
@ -61,6 +63,7 @@ func initObjectPutCmd() {
|
||||||
flags.Bool(binaryFlag, false, "Deserialize object structure from given file.")
|
flags.Bool(binaryFlag, false, "Deserialize object structure from given file.")
|
||||||
|
|
||||||
flags.String(copiesNumberFlag, "", "Number of copies of the object to store within the RPC call")
|
flags.String(copiesNumberFlag, "", "Number of copies of the object to store within the RPC call")
|
||||||
|
flags.String(chunkSizeFlag, "", "The maximum size of payload chunk sended to FrostFS within single request")
|
||||||
}
|
}
|
||||||
|
|
||||||
func putObject(cmd *cobra.Command, _ []string) {
|
func putObject(cmd *cobra.Command, _ []string) {
|
||||||
|
@ -129,6 +132,7 @@ func putObject(cmd *cobra.Command, _ []string) {
|
||||||
copyNum, err := cmd.Flags().GetString(copiesNumberFlag)
|
copyNum, err := cmd.Flags().GetString(copiesNumberFlag)
|
||||||
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
|
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
|
||||||
prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum))
|
prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum))
|
||||||
|
prm.SetGRPCChunkSize(parseChunkSize(cmd))
|
||||||
|
|
||||||
res, err := internalclient.PutObject(cmd.Context(), prm)
|
res, err := internalclient.PutObject(cmd.Context(), prm)
|
||||||
if p != nil {
|
if p != nil {
|
||||||
|
@ -286,3 +290,12 @@ func parseObjectNotifications(cmd *cobra.Command) (*object.NotificationInfo, err
|
||||||
|
|
||||||
return ni, nil
|
return ni, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func parseChunkSize(cmd *cobra.Command) int {
|
||||||
|
raw, err := cmd.Flags().GetString(chunkSizeFlag)
|
||||||
|
commonCmd.ExitOnErr(cmd, "failed to get chunk size flag value: %w", err)
|
||||||
|
if raw == "" {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return int(config.ParseSizeInBytes(raw))
|
||||||
|
}
|
||||||
|
|
|
@ -158,7 +158,7 @@ func IntSafe(c *Config, name string) int64 {
|
||||||
// Returns 0 if a value can't be casted.
|
// Returns 0 if a value can't be casted.
|
||||||
func SizeInBytesSafe(c *Config, name string) uint64 {
|
func SizeInBytesSafe(c *Config, name string) uint64 {
|
||||||
s := StringSafe(c, name)
|
s := StringSafe(c, name)
|
||||||
return parseSizeInBytes(s)
|
return ParseSizeInBytes(s)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The following code is taken from https://github.com/spf13/viper/blob/master/util.go
|
// The following code is taken from https://github.com/spf13/viper/blob/master/util.go
|
||||||
|
@ -189,8 +189,8 @@ func safeMul(size float64, multiplier uint64) uint64 {
|
||||||
return lo
|
return lo
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseSizeInBytes converts strings like 1GB or 12 mb into an unsigned integer number of bytes.
|
// ParseSizeInBytes converts strings like 1GB or 12 mb into an unsigned integer number of bytes.
|
||||||
func parseSizeInBytes(sizeStr string) uint64 {
|
func ParseSizeInBytes(sizeStr string) uint64 {
|
||||||
sizeStr = strings.TrimSpace(sizeStr)
|
sizeStr = strings.TrimSpace(sizeStr)
|
||||||
lastChar := len(sizeStr) - 1
|
lastChar := len(sizeStr) - 1
|
||||||
multiplier := uint64(1)
|
multiplier := uint64(1)
|
||||||
|
|
Loading…
Reference in a new issue