cli: Allow to split object on the client side #471

Merged
fyrchik merged 1 commit from acid-ant/frostfs-node:feature/cut-obj-client into master 2023-07-26 21:07:59 +00:00
2 changed files with 70 additions and 30 deletions
Showing only changes of commit 4c248d573e - Show all commits

View file

@ -336,6 +336,8 @@ type PutObjectPrm struct {
rdr io.Reader
headerCallback func(*object.Object)
prepareLocally bool
}
// SetHeader sets object header.
@ -360,6 +362,41 @@ func (x *PutObjectPrm) SetCopiesNumberByVectors(copiesNumbers []uint32) {
x.copyNum = copiesNumbers
}
// PrepareLocally generate object header on the client side.
// For big object - split locally too.
func (x *PutObjectPrm) PrepareLocally() {
x.prepareLocally = true
}
func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) {
var putPrm client.PrmObjectPutInit
if !x.prepareLocally && x.sessionToken != nil {
fyrchik marked this conversation as resolved Outdated

We might avoid creating a session in this case: it is needed when object header is changed, not the case here.

We might avoid creating a session in this case: it is needed when object header is changed, not the case here.

Looks like we don't need to read it at all when generating header locally.

Looks like we don't need to read it at all when generating header locally.
putPrm.WithinSession(*x.sessionToken)
}
if x.bearerToken != nil {
putPrm.WithBearerToken(*x.bearerToken)
}
if x.local {
putPrm.MarkLocal()
}
putPrm.WithXHeaders(x.xHeaders...)
putPrm.SetCopiesNumberByVectors(x.copyNum)
if x.prepareLocally {
res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{})
if err != nil {
return client.PrmObjectPutInit{}, err
}
putPrm.WithObjectMaxSize(res.Info().MaxObjectSize())
putPrm.WithEpochSource(epochSource(res.Info().CurrentEpoch()))
fyrchik marked this conversation as resolved Outdated

This is also a network setting, not a constant.

This is also a network setting, not a constant.

My fault, fixed.

My fault, fixed.
putPrm.WithoutHomomorphicHash(res.Info().HomomorphicHashingDisabled())
}
return putPrm, nil
}
// PutObjectRes groups the resulting values of PutObject operation.
type PutObjectRes struct {
id oid.ID
@ -370,28 +407,21 @@ func (x PutObjectRes) ID() oid.ID {
return x.id
}
type epochSource uint64
func (s epochSource) CurrentEpoch() uint64 {
return uint64(s)
}
// PutObject saves the object in FrostFS network.
//
// Returns any error which prevented the operation from completing correctly in error return.
func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) {
var putPrm client.PrmObjectPutInit
if prm.sessionToken != nil {
putPrm.WithinSession(*prm.sessionToken)
sdkPrm, err := prm.convertToSDKPrm(ctx)
if err != nil {
return nil, fmt.Errorf("unable to create parameters of object put operation: %w", err)
}
if prm.bearerToken != nil {
putPrm.WithBearerToken(*prm.bearerToken)
}
if prm.local {
putPrm.MarkLocal()
}
putPrm.WithXHeaders(prm.xHeaders...)
putPrm.SetCopiesNumberByVectors(prm.copyNum)
wrt, err := prm.cli.ObjectPutInit(ctx, putPrm)
wrt, err := prm.cli.ObjectPutInit(ctx, sdkPrm)
if err != nil {
return nil, fmt.Errorf("init object writing: %w", err)
}

View file

@ -23,9 +23,10 @@ import (
)
const (
noProgressFlag = "no-progress"
notificationFlag = "notify"
copiesNumberFlag = "copies-number"
noProgressFlag = "no-progress"
notificationFlag = "notify"
copiesNumberFlag = "copies-number"
prepareLocallyFlag = "prepare-locally"
)
var putExpiredOn uint64
@ -54,6 +55,7 @@ func initObjectPutCmd() {
flags.Bool("disable-timestamp", false, "Do not set well-known timestamp attribute")
flags.Uint64VarP(&putExpiredOn, commonflags.ExpireAt, "e", 0, "The last active epoch in the life of the object")
flags.Bool(noProgressFlag, false, "Do not show progress bar")
flags.Bool(prepareLocallyFlag, false, "Generate object header on the client side (for big object - split locally too)")
fyrchik marked this conversation as resolved Outdated

It's not only this, small objects are not split but prepared.
May be "prepare object header(s) on the client"?

It's not only this, small objects are not split but prepared. May be "prepare object header(s) on the client"?

Right, how about this:
Generate object header on the client side (for big object - split locally too)
and option name - gen-locally

Right, how about this: `Generate object header on the client side (for big object - split locally too)` and option name - `gen-locally`
fyrchik marked this conversation as resolved Outdated

Maybe prepare-locally?

Maybe `prepare-locally`?

Why not, updated.

Why not, updated.
flags.String(notificationFlag, "", "Object notification in the form of *epoch*:*topic*; '-' topic means using default")
flags.Bool(binaryFlag, false, "Deserialize object structure from given file.")
@ -102,7 +104,11 @@ func putObject(cmd *cobra.Command, _ []string) {
}
var prm internalclient.PutObjectPrm
ReadOrOpenSession(cmd, &prm, pk, cnr, nil)
if prepareLocally, _ := cmd.Flags().GetBool(prepareLocallyFlag); prepareLocally {
prm.PrepareLocally()
} else {
ReadOrOpenSession(cmd, &prm, pk, cnr, nil)
}
Prepare(cmd, &prm)
prm.SetHeader(obj)
@ -121,15 +127,7 @@ func putObject(cmd *cobra.Command, _ []string) {
copyNum, err := cmd.Flags().GetString(copiesNumberFlag)
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
if len(copyNum) > 0 {
var cn []uint32
for _, num := range strings.Split(copyNum, ",") {
val, err := strconv.ParseUint(num, 10, 32)
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
cn = append(cn, uint32(val))
}
prm.SetCopiesNumberByVectors(cn)
}
prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum))
res, err := internalclient.PutObject(cmd.Context(), prm)
if p != nil {
@ -141,6 +139,18 @@ func putObject(cmd *cobra.Command, _ []string) {
cmd.Printf(" OID: %s\n CID: %s\n", res.ID(), cnr)
}
func parseCopyNumber(cmd *cobra.Command, copyNum string) []uint32 {
var cn []uint32
if len(copyNum) > 0 {
for _, num := range strings.Split(copyNum, ",") {
val, err := strconv.ParseUint(num, 10, 32)
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
cn = append(cn, uint32(val))
}
}
return cn
}
func readFilePayload(filename string, cmd *cobra.Command) (io.Reader, cid.ID, user.ID) {
buf, err := os.ReadFile(filename)
commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err)