cli: Allow to split object on the client side #471

Merged
fyrchik merged 1 commit from acid-ant/frostfs-node:feature/cut-obj-client into master 2023-07-26 21:07:59 +00:00
2 changed files with 70 additions and 30 deletions
Showing only changes of commit 4c248d573e - Show all commits

View file

@ -336,6 +336,8 @@ type PutObjectPrm struct {
rdr io.Reader rdr io.Reader
headerCallback func(*object.Object) headerCallback func(*object.Object)
prepareLocally bool
} }
// SetHeader sets object header. // SetHeader sets object header.
@ -360,6 +362,41 @@ func (x *PutObjectPrm) SetCopiesNumberByVectors(copiesNumbers []uint32) {
x.copyNum = copiesNumbers x.copyNum = copiesNumbers
} }
// PrepareLocally generate object header on the client side.
// For big object - split locally too.
func (x *PutObjectPrm) PrepareLocally() {
x.prepareLocally = true
}
func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) {
var putPrm client.PrmObjectPutInit
if !x.prepareLocally && x.sessionToken != nil {
fyrchik marked this conversation as resolved Outdated

We might avoid creating a session in this case: it is needed when object header is changed, not the case here.

We might avoid creating a session in this case: it is needed when object header is changed, not the case here.

Looks like we don't need to read it at all when generating header locally.

Looks like we don't need to read it at all when generating header locally.
putPrm.WithinSession(*x.sessionToken)
}
if x.bearerToken != nil {
putPrm.WithBearerToken(*x.bearerToken)
}
if x.local {
putPrm.MarkLocal()
}
putPrm.WithXHeaders(x.xHeaders...)
putPrm.SetCopiesNumberByVectors(x.copyNum)
if x.prepareLocally {
res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{})
if err != nil {
return client.PrmObjectPutInit{}, err
}
putPrm.WithObjectMaxSize(res.Info().MaxObjectSize())
putPrm.WithEpochSource(epochSource(res.Info().CurrentEpoch()))
fyrchik marked this conversation as resolved Outdated

This is also a network setting, not a constant.

This is also a network setting, not a constant.

My fault, fixed.

My fault, fixed.
putPrm.WithoutHomomorphicHash(res.Info().HomomorphicHashingDisabled())
}
return putPrm, nil
}
// PutObjectRes groups the resulting values of PutObject operation. // PutObjectRes groups the resulting values of PutObject operation.
type PutObjectRes struct { type PutObjectRes struct {
id oid.ID id oid.ID
@ -370,28 +407,21 @@ func (x PutObjectRes) ID() oid.ID {
return x.id return x.id
} }
type epochSource uint64
func (s epochSource) CurrentEpoch() uint64 {
return uint64(s)
}
// PutObject saves the object in FrostFS network. // PutObject saves the object in FrostFS network.
// //
// Returns any error which prevented the operation from completing correctly in error return. // Returns any error which prevented the operation from completing correctly in error return.
func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) {
var putPrm client.PrmObjectPutInit sdkPrm, err := prm.convertToSDKPrm(ctx)
if err != nil {
if prm.sessionToken != nil { return nil, fmt.Errorf("unable to create parameters of object put operation: %w", err)
putPrm.WithinSession(*prm.sessionToken)
} }
wrt, err := prm.cli.ObjectPutInit(ctx, sdkPrm)
if prm.bearerToken != nil {
putPrm.WithBearerToken(*prm.bearerToken)
}
if prm.local {
putPrm.MarkLocal()
}
putPrm.WithXHeaders(prm.xHeaders...)
putPrm.SetCopiesNumberByVectors(prm.copyNum)
wrt, err := prm.cli.ObjectPutInit(ctx, putPrm)
if err != nil { if err != nil {
return nil, fmt.Errorf("init object writing: %w", err) return nil, fmt.Errorf("init object writing: %w", err)
} }

View file

@ -26,6 +26,7 @@ const (
noProgressFlag = "no-progress" noProgressFlag = "no-progress"
notificationFlag = "notify" notificationFlag = "notify"
copiesNumberFlag = "copies-number" copiesNumberFlag = "copies-number"
prepareLocallyFlag = "prepare-locally"
) )
var putExpiredOn uint64 var putExpiredOn uint64
@ -54,6 +55,7 @@ func initObjectPutCmd() {
flags.Bool("disable-timestamp", false, "Do not set well-known timestamp attribute") flags.Bool("disable-timestamp", false, "Do not set well-known timestamp attribute")
flags.Uint64VarP(&putExpiredOn, commonflags.ExpireAt, "e", 0, "The last active epoch in the life of the object") flags.Uint64VarP(&putExpiredOn, commonflags.ExpireAt, "e", 0, "The last active epoch in the life of the object")
flags.Bool(noProgressFlag, false, "Do not show progress bar") flags.Bool(noProgressFlag, false, "Do not show progress bar")
flags.Bool(prepareLocallyFlag, false, "Generate object header on the client side (for big object - split locally too)")
fyrchik marked this conversation as resolved Outdated

It's not only this, small objects are not split but prepared.
May be "prepare object header(s) on the client"?

It's not only this, small objects are not split but prepared. May be "prepare object header(s) on the client"?

Right, how about this:
Generate object header on the client side (for big object - split locally too)
and option name - gen-locally

Right, how about this: `Generate object header on the client side (for big object - split locally too)` and option name - `gen-locally`
fyrchik marked this conversation as resolved Outdated

Maybe prepare-locally?

Maybe `prepare-locally`?

Why not, updated.

Why not, updated.
flags.String(notificationFlag, "", "Object notification in the form of *epoch*:*topic*; '-' topic means using default") flags.String(notificationFlag, "", "Object notification in the form of *epoch*:*topic*; '-' topic means using default")
flags.Bool(binaryFlag, false, "Deserialize object structure from given file.") flags.Bool(binaryFlag, false, "Deserialize object structure from given file.")
@ -102,7 +104,11 @@ func putObject(cmd *cobra.Command, _ []string) {
} }
var prm internalclient.PutObjectPrm var prm internalclient.PutObjectPrm
if prepareLocally, _ := cmd.Flags().GetBool(prepareLocallyFlag); prepareLocally {
prm.PrepareLocally()
} else {
ReadOrOpenSession(cmd, &prm, pk, cnr, nil) ReadOrOpenSession(cmd, &prm, pk, cnr, nil)
}
Prepare(cmd, &prm) Prepare(cmd, &prm)
prm.SetHeader(obj) prm.SetHeader(obj)
@ -121,15 +127,7 @@ func putObject(cmd *cobra.Command, _ []string) {
copyNum, err := cmd.Flags().GetString(copiesNumberFlag) copyNum, err := cmd.Flags().GetString(copiesNumberFlag)
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
if len(copyNum) > 0 { prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum))
var cn []uint32
for _, num := range strings.Split(copyNum, ",") {
val, err := strconv.ParseUint(num, 10, 32)
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
cn = append(cn, uint32(val))
}
prm.SetCopiesNumberByVectors(cn)
}
res, err := internalclient.PutObject(cmd.Context(), prm) res, err := internalclient.PutObject(cmd.Context(), prm)
if p != nil { if p != nil {
@ -141,6 +139,18 @@ func putObject(cmd *cobra.Command, _ []string) {
cmd.Printf(" OID: %s\n CID: %s\n", res.ID(), cnr) cmd.Printf(" OID: %s\n CID: %s\n", res.ID(), cnr)
} }
func parseCopyNumber(cmd *cobra.Command, copyNum string) []uint32 {
var cn []uint32
if len(copyNum) > 0 {
for _, num := range strings.Split(copyNum, ",") {
val, err := strconv.ParseUint(num, 10, 32)
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
cn = append(cn, uint32(val))
}
}
return cn
}
func readFilePayload(filename string, cmd *cobra.Command) (io.Reader, cid.ID, user.ID) { func readFilePayload(filename string, cmd *cobra.Command) (io.Reader, cid.ID, user.ID) {
buf, err := os.ReadFile(filename) buf, err := os.ReadFile(filename)
commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err) commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err)