carpawell/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#1369] checksum: Upgrade SDK package

Browse source 
Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2022-05-11 19:35:01 +03:00 committed by LeL
parent 088df0e2a9
commit ae92074272
16 changed files with 96 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
cmd/neofs-cli/modules/object.go
View file

@ -681,15 +681,20 @@ func getObjectHash(cmd *cobra.Command, _ []string) {
res, err := internalclient.HeadObject(headPrm)
exitOnErr(cmd, errf("rpc error: %w", err))
var cs *checksum.Checksum
var cs checksum.Checksum
var csSet bool
if tz {
cs = res.Header().PayloadHomomorphicHash()
cs, csSet = res.Header().PayloadHomomorphicHash()
} else {
cs = res.Header().PayloadChecksum()
cs, csSet = res.Header().PayloadChecksum()
}
cmd.Println(hex.EncodeToString(cs.Sum()))
if csSet {
cmd.Println(hex.EncodeToString(cs.Value()))
} else {
cmd.Println("Missing checksum in object header.")
}
return
}
@ -959,14 +964,27 @@ func saveAndPrintHeader(cmd *cobra.Command, obj *object.Object, filename string)
return printHeader(cmd, obj)
}
func printChecksum(cmd *cobra.Command, name string, recv func() (checksum.Checksum, bool)) {
var strVal string
cs, csSet := recv()
if csSet {
strVal = hex.EncodeToString(cs.Value())
} else {
strVal = "<empty>"
}
cmd.Printf("%s: %s\n", name, strVal)
}
func printHeader(cmd *cobra.Command, obj *object.Object) error {
cmd.Printf("ID: %s\n", obj.ID())
cmd.Printf("CID: %s\n", obj.ContainerID())
cmd.Printf("Owner: %s\n", obj.OwnerID())
cmd.Printf("CreatedAt: %d\n", obj.CreationEpoch())
cmd.Printf("Size: %d\n", obj.PayloadSize())
cmd.Printf("HomoHash: %s\n", hex.EncodeToString(obj.PayloadHomomorphicHash().Sum()))
cmd.Printf("Checksum: %s\n", hex.EncodeToString(obj.PayloadChecksum().Sum()))
printChecksum(cmd, "HomoHash", obj.PayloadHomomorphicHash)
printChecksum(cmd, "Checksum", obj.PayloadChecksum)
cmd.Printf("Type: %s\n", obj.Type())
cmd.Println("Attributes:")

2
cmd/neofs-cli/modules/storagegroup.go
View file

@ -255,7 +255,7 @@ func getSG(cmd *cobra.Command, _ []string) {
cmd.Printf("Expiration epoch: %d\n", sg.ExpirationEpoch())
cmd.Printf("Group size: %d\n", sg.ValidationDataSize())
cmd.Printf("Group hash: %s\n", sg.ValidationDataHash())
printChecksum(cmd, "Group hash", sg.ValidationDataHash)
if members := sg.Members(); len(members) > 0 {
cmd.Println("Members:")

2
go.mod
View file

@ -19,7 +19,7 @@ require (
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220321144137-d5a9af5860af // indirect
github.com/nspcc-dev/neofs-api-go/v2 v2.12.1
github.com/nspcc-dev/neofs-contract v0.14.2
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220413072812-c961aea14446
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220413075357-96892d7bc4a8
github.com/nspcc-dev/tzhash v1.5.2
github.com/panjf2000/ants/v2 v2.4.0
github.com/paulmach/orb v0.2.2

2
go.sum
View file

@ -407,6 +407,8 @@ github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:d
github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220413072812-c961aea14446 h1:B8bTBqZvDJKKes+W3C/car7RARgAuvMZNx361zCXPDk=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220413072812-c961aea14446/go.mod h1:Hl7a1l0ntZ4b1ZABpGX6fuAuFS3c6+hyMCUNVvZv/w4=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220413075357-96892d7bc4a8 h1:/nV57s9EQO8JXOHNL4UIv8nT76vN57yPT9aCeCPszLg=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220413075357-96892d7bc4a8/go.mod h1:cQKdlr9Gmp5jxbOJ78S714i1AycfYUzpVddxVUD48WM=
github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE=
github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=

6
pkg/local_object_storage/engine/engine_test.go
View file

@ -152,11 +152,11 @@ func generateObjectWithCID(t testing.TB, cid *cid.ID) *object.Object {
ver.SetMajor(2)
ver.SetMinor(1)
csum := new(checksum.Checksum)
var csum checksum.Checksum
csum.SetSHA256(sha256.Sum256(owner.PublicKeyToIDBytes(&test.DecodeKey(-1).PublicKey)))
csumTZ := new(checksum.Checksum)
csumTZ.SetTillichZemor(tz.Sum(csum.Sum()))
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := object.New()
obj.SetID(testOID())

6
pkg/local_object_storage/metabase/db_test.go
View file

@ -72,11 +72,11 @@ func generateObjectWithCID(t testing.TB, cid *cid.ID) *object.Object {
ver.SetMajor(2)
ver.SetMinor(1)
csum := new(checksum.Checksum)
var csum checksum.Checksum
csum.SetSHA256(sha256.Sum256(owner.PublicKeyToIDBytes(&test.DecodeKey(-1).PublicKey)))
csumTZ := new(checksum.Checksum)
csumTZ.SetTillichZemor(tz.Sum(csum.Sum()))
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := object.New()
obj.SetID(testOID())

4
pkg/local_object_storage/metabase/put.go
View file

@ -247,10 +247,12 @@ func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun
addr := object.AddressOf(obj)
objKey := objectKey(addr.ObjectID())
cs, _ := obj.PayloadChecksum()
// index payload hashes
err := f(tx, namedBucketItem{
name: payloadHashBucketName(addr.ContainerID()),
key: obj.PayloadChecksum().Sum(),
key: cs.Value(),
val: objKey,
})
if err != nil {

3
pkg/local_object_storage/metabase/select.go
View file

@ -516,7 +516,8 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr *addressSDK.Address, f object.
case v2object.FilterHeaderVersion:
data = []byte(obj.Version().String())
case v2object.FilterHeaderHomomorphicHash:
data = obj.PayloadHomomorphicHash().Sum()
cs, _ := obj.PayloadHomomorphicHash()
data = cs.Value()
case v2object.FilterHeaderCreationEpoch:
data = make([]byte, 8)
binary.LittleEndian.PutUint64(data, obj.CreationEpoch())

12
pkg/local_object_storage/metabase/select_test.go
View file

@ -358,16 +358,18 @@ func TestDB_SelectPayloadHash(t *testing.T) {
err = putBig(db, raw2)
require.NoError(t, err)
cs, _ := raw1.PayloadChecksum()
fs := objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderPayloadHash,
hex.EncodeToString(raw1.PayloadChecksum().Sum()),
hex.EncodeToString(cs.Value()),
objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, object.AddressOf(raw1))
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderPayloadHash,
hex.EncodeToString(raw1.PayloadChecksum().Sum()),
hex.EncodeToString(cs.Value()),
objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs, object.AddressOf(raw2))
@ -407,16 +409,18 @@ func TestDB_SelectWithSlowFilters(t *testing.T) {
require.NoError(t, err)
t.Run("object with TZHash", func(t *testing.T) {
cs, _ := raw1.PayloadHomomorphicHash()
fs := objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderHomomorphicHash,
hex.EncodeToString(raw1.PayloadHomomorphicHash().Sum()),
hex.EncodeToString(cs.Value()),
objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, object.AddressOf(raw1))
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderHomomorphicHash,
hex.EncodeToString(raw1.PayloadHomomorphicHash().Sum()),
hex.EncodeToString(cs.Value()),
objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs, object.AddressOf(raw2))

6
pkg/local_object_storage/shard/shard_test.go
View file

@ -86,11 +86,11 @@ func generateObjectWithPayload(cid *cid.ID, data []byte) *object.Object {
ver.SetMajor(2)
ver.SetMinor(1)
csum := new(checksum.Checksum)
var csum checksum.Checksum
csum.SetSHA256(sha256.Sum256(data))
csumTZ := new(checksum.Checksum)
csumTZ.SetTillichZemor(tz.Sum(csum.Sum()))
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := object.New()
obj.SetID(generateOID())

4
pkg/services/audit/auditor/context.go
View file

@ -256,9 +256,11 @@ func (c *Context) updateHeadResponses(hdr *object.Object) {
strID := hdr.ID().String()
cs, _ := hdr.PayloadHomomorphicHash()
if _, ok := c.headResponses[strID]; !ok {
c.headResponses[strID] = shortHeader{
tzhash: hdr.PayloadHomomorphicHash().Sum(),
tzhash: cs.Value(),
objectSize: hdr.PayloadSize(),
}
}

12
pkg/services/audit/auditor/por.go
View file

@ -90,13 +90,12 @@ func (c *Context) checkStorageGroupPoR(ind int, sg *oidSDK.ID) {
// update cache for PoR and PDP audit checks
c.updateHeadResponses(hdr)
cs, _ := hdr.PayloadHomomorphicHash()
if len(tzHash) == 0 {
tzHash = hdr.PayloadHomomorphicHash().Sum()
tzHash = cs.Value()
} else {
tzHash, err = tz.Concat([][]byte{
tzHash,
hdr.PayloadHomomorphicHash().Sum(),
})
tzHash, err = tz.Concat([][]byte{tzHash, cs.Value()})
if err != nil {
c.log.Debug("can't concatenate tz hash",
zap.String("oid", members[i].String()),
@ -116,7 +115,8 @@ func (c *Context) checkStorageGroupPoR(ind int, sg *oidSDK.ID) {
c.porRetries.Add(accRetries)
sizeCheck := storageGroup.ValidationDataSize() == totalSize
tzCheck := bytes.Equal(tzHash, storageGroup.ValidationDataHash().Sum())
cs, _ := storageGroup.ValidationDataHash()
tzCheck := bytes.Equal(tzHash, cs.Value())
if sizeCheck && tzCheck {
c.report.PassedPoR(sg) // write report

26
pkg/services/object/acl/eacl/v2/object.go
View file

@ -71,23 +71,29 @@ func headersFromObject(obj *object.Object, addr *objectSDKAddress.Address) []eac
k: acl.FilterObjectVersion,
v: obj.Version().String(),
},
// payload hash
&sysObjHdr{
k: acl.FilterObjectPayloadHash,
v: obj.PayloadChecksum().String(),
},
// object type
&sysObjHdr{
k: acl.FilterObjectType,
v: obj.Type().String(),
},
// payload homomorphic hash
&sysObjHdr{
k: acl.FilterObjectHomomorphicHash,
v: obj.PayloadHomomorphicHash().String(),
},
)
cs, ok := obj.PayloadChecksum()
if ok {
res = append(res, &sysObjHdr{
k: acl.FilterObjectPayloadHash,
v: cs.String(),
})
}
cs, ok = obj.PayloadHomomorphicHash()
if ok {
res = append(res, &sysObjHdr{
k: acl.FilterObjectHomomorphicHash,
v: cs.String(),
})
}
attrs := obj.Attributes()
for i := range attrs {
res = append(res, &attrs[i]) // only pointer attrs can implement eaclSDK.Header interface

8
pkg/services/object/put/validation.go
View file

@ -54,7 +54,11 @@ func (t *validatingTarget) WriteHeader(obj *objectSDK.Object) error {
return ErrExceedingMaxSize
}
cs := obj.PayloadChecksum()
cs, csSet := obj.PayloadChecksum()
if !csSet {
return errors.New("missing payload checksum")
}
switch typ := cs.Type(); typ {
default:
return fmt.Errorf("(%T) unsupported payload checksum type %v", t, typ)
@ -64,7 +68,7 @@ func (t *validatingTarget) WriteHeader(obj *objectSDK.Object) error {
t.hash = tz.New()
}
t.checksum = cs.Sum()
t.checksum = cs.Value()
}
if err := t.fmt.Validate(obj, t.unpreparedObject); err != nil {

5
pkg/services/object_manager/storagegroup/collect.go
View file

@ -32,7 +32,8 @@ func CollectMembers(r objutil.HeadReceiver, cid *cid.ID, members []oidSDK.ID) (*
if err := objutil.IterateAllSplitLeaves(r, addr, func(leaf *object.Object) {
phyMembers = append(phyMembers, *leaf.ID())
sumPhySize += leaf.PayloadSize()
phyHashes = append(phyHashes, leaf.PayloadHomomorphicHash().Sum())
cs, _ := leaf.PayloadHomomorphicHash()
phyHashes = append(phyHashes, cs.Value())
}); err != nil {
return nil, err
}
@ -43,7 +44,7 @@ func CollectMembers(r objutil.HeadReceiver, cid *cid.ID, members []oidSDK.ID) (*
return nil, err
}
cs := checksum.New()
var cs checksum.Checksum
tzHash := [64]byte{}
copy(tzHash[:], sumHash)
cs.SetTillichZemor(tzHash)

24
pkg/services/object_manager/transformer/transformer.go
View file

@ -128,34 +128,34 @@ func payloadHashersForObject(obj *object.Object) []*payloadChecksumHasher {
return []*payloadChecksumHasher{
{
hasher: sha256.New(),
checksumWriter: func(cs []byte) {
if ln := len(cs); ln != sha256.Size {
checksumWriter: func(binChecksum []byte) {
if ln := len(binChecksum); ln != sha256.Size {
panic(fmt.Sprintf("wrong checksum length: expected %d, has %d", sha256.Size, ln))
}
csSHA := [sha256.Size]byte{}
copy(csSHA[:], cs)
copy(csSHA[:], binChecksum)
checksum := checksum.New()
checksum.SetSHA256(csSHA)
var cs checksum.Checksum
cs.SetSHA256(csSHA)
obj.SetPayloadChecksum(checksum)
obj.SetPayloadChecksum(cs)
},
},
{
hasher: tz.New(),
checksumWriter: func(cs []byte) {
if ln := len(cs); ln != tz.Size {
checksumWriter: func(binChecksum []byte) {
if ln := len(binChecksum); ln != tz.Size {
panic(fmt.Sprintf("wrong checksum length: expected %d, has %d", tz.Size, ln))
}
csTZ := [tz.Size]byte{}
copy(csTZ[:], cs)
copy(csTZ[:], binChecksum)
checksum := checksum.New()
checksum.SetTillichZemor(csTZ)
var cs checksum.Checksum
cs.SetTillichZemor(csTZ)
obj.SetPayloadHomomorphicHash(checksum)
obj.SetPayloadHomomorphicHash(cs)
},
},
}