From 247ee831e5eb6d4ca87397ca367fbbddf2c6f6c3 Mon Sep 17 00:00:00 2001 From: Ekaterina Pavlova Date: Thu, 12 Dec 2024 20:03:24 +0300 Subject: [PATCH 1/2] cli: change block timestamp attribute of objects in `upload-bin` Close #3654 Signed-off-by: Ekaterina Pavlova --- cli/util/upload_bin.go | 4 +++- docs/neofs-blockstorage.md | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/util/upload_bin.go b/cli/util/upload_bin.go index 274bcad31..90dff60c5 100644 --- a/cli/util/upload_bin.go +++ b/cli/util/upload_bin.go @@ -211,7 +211,8 @@ func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.C *object.NewAttribute("Primary", strconv.Itoa(int(blk.PrimaryIndex))), *object.NewAttribute("Hash", blk.Hash().StringLE()), *object.NewAttribute("PrevHash", blk.PrevHash.StringLE()), - *object.NewAttribute("Timestamp", strconv.FormatUint(blk.Timestamp, 10)), + *object.NewAttribute("BlockTime", strconv.FormatUint(blk.Timestamp, 10)), + *object.NewAttribute("Timestamp", strconv.FormatInt(time.Now().Unix(), 10)), } var ( @@ -263,6 +264,7 @@ func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.C attrs := []object.Attribute{ *object.NewAttribute(indexAttributeKey, strconv.Itoa(int(indexFileStart/indexFileSize))), *object.NewAttribute("IndexSize", strconv.Itoa(int(indexFileSize))), + *object.NewAttribute("Timestamp", strconv.FormatInt(time.Now().Unix(), 10)), } err := retry(func() error { var errUpload error diff --git a/docs/neofs-blockstorage.md b/docs/neofs-blockstorage.md index fe71b233e..f1115d95e 100644 --- a/docs/neofs-blockstorage.md +++ b/docs/neofs-blockstorage.md @@ -16,13 +16,15 @@ form as a separate object with a unique OID and a set of attributes: - primary node index (`Primary:0`) - block hash in the LE form (`Hash:5412a781caf278c0736556c0e544c7cfdbb6e3c62ae221ef53646be89364566b`) - previous block hash in the LE form (`PrevHash:3654a054d82a8178c7dfacecc2c57282e23468a42ee407f14506368afe22d929`) - - millisecond-precision block timestamp (`Timestamp:1627894840919`) + - millisecond-precision block creation timestamp (`BlockTime:1627894840919`) + - second-precision block uploading timestamp (`Timestamp:1627894840`) Each index file is an object containing a constant-sized batch of raw block object IDs in binary form ordered by block index. Each index file is marked with the following attributes: - index file identifier with consecutive file index value (`Index:0`) - the number of OIDs included into index file (`IndexSize:128000`) + - second-precision index file uploading timestamp (`Timestamp:1627894840`) ### NeoFS BlockFetcher From be4fc9804117e8edff96ed982016434c62e09a44 Mon Sep 17 00:00:00 2001 From: Ekaterina Pavlova Date: Thu, 12 Dec 2024 20:15:53 +0300 Subject: [PATCH 2/2] cli: refactor `upload-bin` handler to reuse SDK built-in functionality Close #3714 Signed-off-by: Ekaterina Pavlova --- cli/util/upload_bin.go | 46 +++++------------------------------------- 1 file changed, 5 insertions(+), 41 deletions(-) diff --git a/cli/util/upload_bin.go b/cli/util/upload_bin.go index 90dff60c5..5fec7bc45 100644 --- a/cli/util/upload_bin.go +++ b/cli/util/upload_bin.go @@ -16,16 +16,13 @@ import ( "github.com/nspcc-dev/neo-go/pkg/services/helpers/neofs" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" - "github.com/nspcc-dev/neofs-sdk-go/checksum" "github.com/nspcc-dev/neofs-sdk-go/client" "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/nspcc-dev/neofs-sdk-go/user" - "github.com/nspcc-dev/neofs-sdk-go/version" "github.com/urfave/cli/v2" ) @@ -86,17 +83,6 @@ func uploadBin(ctx *cli.Context) error { } defer p.Close() - var net netmap.NetworkInfo - err = retry(func() error { - var errNet error - net, errNet = p.NetworkInfo(ctx.Context, client.PrmNetworkInfo{}) - return errNet - }, maxRetries, debug) - if err != nil { - return cli.Exit(fmt.Errorf("failed to get network info: %w", err), 1) - } - homomorphicHashingDisabled := net.HomomorphicHashingDisabled() - var containerObj container.Container err = retry(func() error { containerObj, err = p.ContainerGet(ctx.Context, containerID, client.PrmContainerGet{}) @@ -126,7 +112,7 @@ func uploadBin(ctx *cli.Context) error { return cli.Exit(fmt.Errorf("failed to find objects: %w", err), 1) } - err = uploadBlocksAndIndexFiles(ctx, pWrapper, rpc, signer, containerID, acc, attr, indexAttrKey, buf, i, indexFileSize, uint(currentBlockHeight), homomorphicHashingDisabled, numWorkers, maxRetries, debug) + err = uploadBlocksAndIndexFiles(ctx, pWrapper, rpc, signer, containerID, acc, attr, indexAttrKey, buf, i, indexFileSize, uint(currentBlockHeight), numWorkers, maxRetries, debug) if err != nil { return cli.Exit(fmt.Errorf("failed to upload objects: %w", err), 1) } @@ -154,7 +140,7 @@ func retry(action func() error, maxRetries uint, debug bool) error { } // uploadBlocksAndIndexFiles uploads the blocks and index files to the container using the pool. -func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.Client, signer user.Signer, containerID cid.ID, acc *wallet.Account, attr, indexAttributeKey string, buf []byte, currentIndexFileID, indexFileSize, currentBlockHeight uint, homomorphicHashingDisabled bool, numWorkers, maxRetries uint, debug bool) error { +func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.Client, signer user.Signer, containerID cid.ID, acc *wallet.Account, attr, indexAttributeKey string, buf []byte, currentIndexFileID, indexFileSize, currentBlockHeight uint, numWorkers, maxRetries uint, debug bool) error { if currentIndexFileID*indexFileSize >= currentBlockHeight { fmt.Fprintf(ctx.App.Writer, "No new blocks to upload. Need to upload starting from %d, current height %d\n", currentIndexFileID*indexFileSize, currentBlockHeight) return nil @@ -221,7 +207,7 @@ func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.C ) errRetr := retry(func() error { var errUpload error - resOid, errUpload = uploadObj(ctx.Context, p, signer, acc.PrivateKey().GetScriptHash(), containerID, objBytes, attrs, homomorphicHashingDisabled) + resOid, errUpload = uploadObj(ctx.Context, p, signer, acc.PrivateKey().GetScriptHash(), containerID, objBytes, attrs) if errUpload != nil { return errUpload } @@ -268,7 +254,7 @@ func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.C } err := retry(func() error { var errUpload error - _, errUpload = uploadObj(ctx.Context, p, signer, acc.PrivateKey().GetScriptHash(), containerID, buf, attrs, homomorphicHashingDisabled) + _, errUpload = uploadObj(ctx.Context, p, signer, acc.PrivateKey().GetScriptHash(), containerID, buf, attrs) return errUpload }, maxRetries, debug) if err != nil { @@ -440,41 +426,19 @@ func searchObjects(ctx context.Context, p poolWrapper, containerID cid.ID, accou } // uploadObj uploads object to the container using provided settings. -func uploadObj(ctx context.Context, p poolWrapper, signer user.Signer, owner util.Uint160, containerID cid.ID, objData []byte, attrs []object.Attribute, homomorphicHashingDisabled bool) (oid.ID, error) { +func uploadObj(ctx context.Context, p poolWrapper, signer user.Signer, owner util.Uint160, containerID cid.ID, objData []byte, attrs []object.Attribute) (oid.ID, error) { var ( ownerID user.ID hdr object.Object - chSHA256 checksum.Checksum - chHomomorphic checksum.Checksum - v = new(version.Version) prmObjectPutInit client.PrmObjectPutInit resOID = oid.ID{} ) ownerID.SetScriptHash(owner) hdr.SetPayload(objData) - hdr.SetPayloadSize(uint64(len(objData))) hdr.SetContainerID(containerID) hdr.SetOwnerID(&ownerID) hdr.SetAttributes(attrs...) - hdr.SetCreationEpoch(1) - v.SetMajor(1) - hdr.SetVersion(v) - if !homomorphicHashingDisabled { - checksum.Calculate(&chHomomorphic, checksum.TZ, objData) - hdr.SetPayloadHomomorphicHash(chHomomorphic) - } - checksum.Calculate(&chSHA256, checksum.SHA256, objData) - hdr.SetPayloadChecksum(chSHA256) - - err := hdr.SetIDWithSignature(signer) - if err != nil { - return resOID, err - } - err = hdr.CheckHeaderVerificationFields() - if err != nil { - return resOID, err - } writer, err := p.ObjectPutInit(ctx, hdr, signer, prmObjectPutInit) if err != nil {