TrueCloudLab/neo-go
8
0
Fork 0
mirror of https://github.com/nspcc-dev/neo-go.git synced 2024-11-22 09:29:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #3637 from nspcc-dev/attributes

Browse source 
BlockFetcher/BlockUploader: fix and add additional attributes
This commit is contained in:
Anna Shaleva 2024-10-23 15:07:09 +03:00 committed by GitHub
commit 0b79901b7f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 46 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
cli/util/uploader.go
View file

@ -18,6 +18,7 @@ import (
"github.com/nspcc-dev/neo-go/pkg/wallet"
"github.com/nspcc-dev/neofs-sdk-go/checksum"
"github.com/nspcc-dev/neofs-sdk-go/client"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
@ -83,12 +84,6 @@ func uploadBin(ctx *cli.Context) error {
return cli.Exit(fmt.Sprintf("failed to create RPC client: %v", err), 1)
}
currentBlockHeight, err := rpc.GetBlockCount()
if err != nil {
return cli.Exit(fmt.Sprintf("failed to get current block height from RPC: %v", err), 1)
}
fmt.Fprintln(ctx.App.Writer, "Chain block height:", currentBlockHeight)
signer := user.NewAutoIDSignerRFC6979(acc.PrivateKey().PrivateKey)
params := pool.DefaultOptions()
@ -109,6 +104,31 @@ func uploadBin(ctx *cli.Context) error {
return cli.Exit(fmt.Errorf("failed to get network info: %w", err), 1)
}
homomorphicHashingDisabled := net.HomomorphicHashingDisabled()
var containerObj container.Container
err = retry(func() error {
containerObj, err = p.ContainerGet(ctx.Context, containerID, client.PrmContainerGet{})
return err
})
if err != nil {
return cli.Exit(fmt.Errorf("failed to get container with ID %s: %w", containerID, err), 1)
}
containerMagic := containerObj.Attribute("Magic")
v, err := rpc.GetVersion()
if err != nil {
return cli.Exit(fmt.Sprintf("failed to get version from RPC: %v", err), 1)
}
magic := strconv.Itoa(int(v.Protocol.Network))
if containerMagic != magic {
return cli.Exit(fmt.Sprintf("Container magic %s does not match the network magic %s", containerMagic, magic), 1)
}
currentBlockHeight, err := rpc.GetBlockCount()
if err != nil {
return cli.Exit(fmt.Sprintf("failed to get current block height from RPC: %v", err), 1)
}
fmt.Fprintln(ctx.App.Writer, "Chain block height:", currentBlockHeight)
lastMissingBlockIndex, err := fetchLatestMissingBlockIndex(ctx.Context, p, containerID, acc.PrivateKey(), attr, int(currentBlockHeight))
if err != nil {
return cli.Exit(fmt.Errorf("failed to fetch the latest missing block index from container: %w", err), 1)
@ -158,10 +178,10 @@ func uploadBin(ctx *cli.Context) error {
}
attrs := []object.Attribute{
*object.NewAttribute(attr, strconv.Itoa(int(blk.Index))),
*object.NewAttribute("primary", strconv.Itoa(int(blk.PrimaryIndex))),
*object.NewAttribute("hash", blk.Hash().StringLE()),
*object.NewAttribute("prevHash", blk.PrevHash.StringLE()),
*object.NewAttribute("timestamp", strconv.FormatUint(blk.Timestamp, 10)),
*object.NewAttribute("Primary", strconv.Itoa(int(blk.PrimaryIndex))),
*object.NewAttribute("Hash", blk.Hash().StringLE()),
*object.NewAttribute("PrevHash", blk.PrevHash.StringLE()),
*object.NewAttribute("Timestamp", strconv.FormatUint(blk.Timestamp, 10)),
}
objBytes := bw.Bytes()
@ -293,7 +313,7 @@ func updateIndexFiles(ctx *cli.Context, p *pool.Pool, containerID cid.ID, accoun
prm := client.PrmObjectSearch{}
filters := object.NewSearchFilters()
filters.AddFilter(attributeKey, fmt.Sprintf("%d", 0), object.MatchNumGE)
filters.AddFilter("size", fmt.Sprintf("%d", indexFileSize), object.MatchStringEqual)
filters.AddFilter("IndexSize", fmt.Sprintf("%d", indexFileSize), object.MatchStringEqual)
prm.SetFilters(filters)
var (
objectIDs []oid.ID
@ -395,7 +415,7 @@ func updateIndexFiles(ctx *cli.Context, p *pool.Pool, containerID cid.ID, accoun
}
attrs := []object.Attribute{
*object.NewAttribute(attributeKey, strconv.Itoa(int(i))),
*object.NewAttribute("size", strconv.Itoa(int(indexFileSize))),
*object.NewAttribute("IndexSize", strconv.Itoa(int(indexFileSize))),
}
err = uploadObj(ctx.Context, p, signer, account.PrivateKey().GetScriptHash(), containerID, buffer, attrs, homomorphicHashingDisabled)
if err != nil {

4
cli/util/util_test.go
View file

@ -208,7 +208,5 @@ func TestUploadBin(t *testing.T) {
e.In.WriteString("one\r")
e.RunWithErrorCheckExit(t, "failed to create RPC client", append(args, "--cid", "9iVfUg8aDHKjPC4LhQXEkVUM4HDkR7UCXYLs8NQwYfSG", "--wallet", testcli.ValidatorWallet, "--rpc-endpoint", "https://test")...)
e.In.WriteString("one\r")
e.RunWithErrorCheck(t, "failed to dial NeoFS pool", append(args, "--cid", "9iVfUg8aDHKjPC4LhQXEkVUM4HDkR7UCXYLs8NQwYfSG", "--wallet", testcli.ValidatorWallet, "--rpc-endpoint", "http://"+e.RPC.Addresses()[0])...)
e.CheckNextLine(t, "Chain block height:")
e.CheckEOF(t)
e.RunWithErrorCheckExit(t, "failed to dial NeoFS pool", append(args, "--cid", "9iVfUg8aDHKjPC4LhQXEkVUM4HDkR7UCXYLs8NQwYfSG", "--wallet", testcli.ValidatorWallet, "--rpc-endpoint", "http://"+e.RPC.Addresses()[0])...)
}

4
config/protocol.testnet.yml
View file

@ -111,5 +111,5 @@ ApplicationConfiguration:
SkipIndexFilesSearch: false
IndexFileSize: 128000
ContainerID: "EPGuD26wYgQJbmDdVBoYoNZiMKHwFMJT3A5WqPjdUHxH"
BlockAttribute: "block"
IndexFileAttribute: "index"
BlockAttribute: "Block"
IndexFileAttribute: "Index"

22
docs/neofs-blockstorage.md
View file

@ -9,20 +9,20 @@ efficiency and reduce node storage size.
### Block storage schema
A single NeoFS container is used to store blocks and index files. Each block
is stored in a binary form as a separate object with a unique OID and a set of
attributes:
- block object identifier with block index value (`block:1`)
- primary node index (`primary:0`)
- block hash in the LE form (`hash:5412a781caf278c0736556c0e544c7cfdbb6e3c62ae221ef53646be89364566b`)
- previous block hash in the LE form (`prevHash:3654a054d82a8178c7dfacecc2c57282e23468a42ee407f14506368afe22d929`)
- millisecond-precision block timestamp (`timestamp:1627894840919`)
A single NeoFS container is used to store blocks and index files. Each container
has network magic attribute (`Magic:56753`). Each block is stored in a binary
form as a separate object with a unique OID and a set of attributes:
- block object identifier with block index value (`Block:1`)
- primary node index (`Primary:0`)
- block hash in the LE form (`Hash:5412a781caf278c0736556c0e544c7cfdbb6e3c62ae221ef53646be89364566b`)
- previous block hash in the LE form (`PrevHash:3654a054d82a8178c7dfacecc2c57282e23468a42ee407f14506368afe22d929`)
- millisecond-precision block timestamp (`Timestamp:1627894840919`)
Each index file is an object containing a constant-sized batch of raw block object
IDs in binary form ordered by block index. Each index file is marked with the
following attributes:
- index file identifier with consecutive file index value (`index:0`)
- the number of OIDs included into index file (`size:128000`)
- index file identifier with consecutive file index value (`Index:0`)
- the number of OIDs included into index file (`IndexSize:128000`)
### NeoFS BlockFetcher
@ -79,7 +79,7 @@ them to the NeoFS container.
It also creates and uploads index files. Below is an example usage of the command:
```shell
./bin/neo-go util upload-bin --cid 9iVfUg8aDHKjPC4LhQXEkVUM4HDkR7UCXYLs8NQwYfSG --wallet-config ./wallet-config.yml --block-attribute block --index-attribute index --rpc-endpoint https://rpc.t5.n3.nspcc.ru:20331 -fsr st1.t5.fs.neo.org:8080 -fsr st2.t5.fs.neo.org:8080 -fsr st3.t5.fs.neo.org:8080
./bin/neo-go util upload-bin --cid 9iVfUg8aDHKjPC4LhQXEkVUM4HDkR7UCXYLs8NQwYfSG --wallet-config ./wallet-config.yml --block-attribute Block --index-attribute Index --rpc-endpoint https://rpc.t5.n3.nspcc.ru:20331 -fsr st1.t5.fs.neo.org:8080 -fsr st2.t5.fs.neo.org:8080 -fsr st3.t5.fs.neo.org:8080
```
The command supports the following options:
```