cli: Allow to split object on the client side #471
2 changed files with 70 additions and 30 deletions
|
@ -336,6 +336,8 @@ type PutObjectPrm struct {
|
|||
rdr io.Reader
|
||||
|
||||
headerCallback func(*object.Object)
|
||||
|
||||
prepareLocally bool
|
||||
}
|
||||
|
||||
// SetHeader sets object header.
|
||||
|
@ -360,6 +362,41 @@ func (x *PutObjectPrm) SetCopiesNumberByVectors(copiesNumbers []uint32) {
|
|||
x.copyNum = copiesNumbers
|
||||
}
|
||||
|
||||
// PrepareLocally generate object header on the client side.
|
||||
// For big object - split locally too.
|
||||
func (x *PutObjectPrm) PrepareLocally() {
|
||||
x.prepareLocally = true
|
||||
}
|
||||
|
||||
func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) {
|
||||
var putPrm client.PrmObjectPutInit
|
||||
if !x.prepareLocally && x.sessionToken != nil {
|
||||
fyrchik marked this conversation as resolved
Outdated
|
||||
putPrm.WithinSession(*x.sessionToken)
|
||||
}
|
||||
|
||||
if x.bearerToken != nil {
|
||||
putPrm.WithBearerToken(*x.bearerToken)
|
||||
}
|
||||
|
||||
if x.local {
|
||||
putPrm.MarkLocal()
|
||||
}
|
||||
|
||||
putPrm.WithXHeaders(x.xHeaders...)
|
||||
putPrm.SetCopiesNumberByVectors(x.copyNum)
|
||||
|
||||
if x.prepareLocally {
|
||||
res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{})
|
||||
if err != nil {
|
||||
return client.PrmObjectPutInit{}, err
|
||||
}
|
||||
putPrm.WithObjectMaxSize(res.Info().MaxObjectSize())
|
||||
putPrm.WithEpochSource(epochSource(res.Info().CurrentEpoch()))
|
||||
fyrchik marked this conversation as resolved
Outdated
fyrchik
commented
This is also a network setting, not a constant. This is also a network setting, not a constant.
acid-ant
commented
My fault, fixed. My fault, fixed.
|
||||
putPrm.WithoutHomomorphicHash(res.Info().HomomorphicHashingDisabled())
|
||||
}
|
||||
return putPrm, nil
|
||||
}
|
||||
|
||||
// PutObjectRes groups the resulting values of PutObject operation.
|
||||
type PutObjectRes struct {
|
||||
id oid.ID
|
||||
|
@ -370,28 +407,21 @@ func (x PutObjectRes) ID() oid.ID {
|
|||
return x.id
|
||||
}
|
||||
|
||||
type epochSource uint64
|
||||
|
||||
func (s epochSource) CurrentEpoch() uint64 {
|
||||
return uint64(s)
|
||||
}
|
||||
|
||||
// PutObject saves the object in FrostFS network.
|
||||
//
|
||||
// Returns any error which prevented the operation from completing correctly in error return.
|
||||
func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) {
|
||||
var putPrm client.PrmObjectPutInit
|
||||
|
||||
if prm.sessionToken != nil {
|
||||
putPrm.WithinSession(*prm.sessionToken)
|
||||
sdkPrm, err := prm.convertToSDKPrm(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create parameters of object put operation: %w", err)
|
||||
}
|
||||
|
||||
if prm.bearerToken != nil {
|
||||
putPrm.WithBearerToken(*prm.bearerToken)
|
||||
}
|
||||
|
||||
if prm.local {
|
||||
putPrm.MarkLocal()
|
||||
}
|
||||
|
||||
putPrm.WithXHeaders(prm.xHeaders...)
|
||||
putPrm.SetCopiesNumberByVectors(prm.copyNum)
|
||||
|
||||
wrt, err := prm.cli.ObjectPutInit(ctx, putPrm)
|
||||
wrt, err := prm.cli.ObjectPutInit(ctx, sdkPrm)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init object writing: %w", err)
|
||||
}
|
||||
|
|
|
@ -23,9 +23,10 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
noProgressFlag = "no-progress"
|
||||
notificationFlag = "notify"
|
||||
copiesNumberFlag = "copies-number"
|
||||
noProgressFlag = "no-progress"
|
||||
notificationFlag = "notify"
|
||||
copiesNumberFlag = "copies-number"
|
||||
prepareLocallyFlag = "prepare-locally"
|
||||
)
|
||||
|
||||
var putExpiredOn uint64
|
||||
|
@ -54,6 +55,7 @@ func initObjectPutCmd() {
|
|||
flags.Bool("disable-timestamp", false, "Do not set well-known timestamp attribute")
|
||||
flags.Uint64VarP(&putExpiredOn, commonflags.ExpireAt, "e", 0, "The last active epoch in the life of the object")
|
||||
flags.Bool(noProgressFlag, false, "Do not show progress bar")
|
||||
flags.Bool(prepareLocallyFlag, false, "Generate object header on the client side (for big object - split locally too)")
|
||||
fyrchik marked this conversation as resolved
Outdated
fyrchik
commented
It's not only this, small objects are not split but prepared. It's not only this, small objects are not split but prepared.
May be "prepare object header(s) on the client"?
acid-ant
commented
Right, how about this: Right, how about this:
`Generate object header on the client side (for big object - split locally too)`
and option name - `gen-locally`
fyrchik marked this conversation as resolved
Outdated
fyrchik
commented
Maybe Maybe `prepare-locally`?
acid-ant
commented
Why not, updated. Why not, updated.
|
||||
|
||||
flags.String(notificationFlag, "", "Object notification in the form of *epoch*:*topic*; '-' topic means using default")
|
||||
flags.Bool(binaryFlag, false, "Deserialize object structure from given file.")
|
||||
|
@ -102,7 +104,11 @@ func putObject(cmd *cobra.Command, _ []string) {
|
|||
}
|
||||
|
||||
var prm internalclient.PutObjectPrm
|
||||
ReadOrOpenSession(cmd, &prm, pk, cnr, nil)
|
||||
if prepareLocally, _ := cmd.Flags().GetBool(prepareLocallyFlag); prepareLocally {
|
||||
prm.PrepareLocally()
|
||||
} else {
|
||||
ReadOrOpenSession(cmd, &prm, pk, cnr, nil)
|
||||
}
|
||||
Prepare(cmd, &prm)
|
||||
prm.SetHeader(obj)
|
||||
|
||||
|
@ -121,15 +127,7 @@ func putObject(cmd *cobra.Command, _ []string) {
|
|||
|
||||
copyNum, err := cmd.Flags().GetString(copiesNumberFlag)
|
||||
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
|
||||
if len(copyNum) > 0 {
|
||||
var cn []uint32
|
||||
for _, num := range strings.Split(copyNum, ",") {
|
||||
val, err := strconv.ParseUint(num, 10, 32)
|
||||
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
|
||||
cn = append(cn, uint32(val))
|
||||
}
|
||||
prm.SetCopiesNumberByVectors(cn)
|
||||
}
|
||||
prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum))
|
||||
|
||||
res, err := internalclient.PutObject(cmd.Context(), prm)
|
||||
if p != nil {
|
||||
|
@ -141,6 +139,18 @@ func putObject(cmd *cobra.Command, _ []string) {
|
|||
cmd.Printf(" OID: %s\n CID: %s\n", res.ID(), cnr)
|
||||
}
|
||||
|
||||
func parseCopyNumber(cmd *cobra.Command, copyNum string) []uint32 {
|
||||
var cn []uint32
|
||||
if len(copyNum) > 0 {
|
||||
for _, num := range strings.Split(copyNum, ",") {
|
||||
val, err := strconv.ParseUint(num, 10, 32)
|
||||
commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err)
|
||||
cn = append(cn, uint32(val))
|
||||
}
|
||||
}
|
||||
return cn
|
||||
}
|
||||
|
||||
func readFilePayload(filename string, cmd *cobra.Command) (io.Reader, cid.ID, user.ID) {
|
||||
buf, err := os.ReadFile(filename)
|
||||
commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err)
|
||||
|
|
Loading…
Reference in a new issue
We might avoid creating a session in this case: it is needed when object header is changed, not the case here.
Looks like we don't need to read it at all when generating header locally.