[#1369] checksum: Upgrade SDK package

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2022-05-11 19:35:01 +03:00 committed by LeL
parent 088df0e2a9
commit ae92074272
16 changed files with 94 additions and 56 deletions

View file

@ -681,15 +681,20 @@ func getObjectHash(cmd *cobra.Command, _ []string) {
res, err := internalclient.HeadObject(headPrm)
exitOnErr(cmd, errf("rpc error: %w", err))
var cs *checksum.Checksum
var cs checksum.Checksum
var csSet bool
if tz {
cs = res.Header().PayloadHomomorphicHash()
cs, csSet = res.Header().PayloadHomomorphicHash()
} else {
cs = res.Header().PayloadChecksum()
cs, csSet = res.Header().PayloadChecksum()
}
cmd.Println(hex.EncodeToString(cs.Sum()))
if csSet {
cmd.Println(hex.EncodeToString(cs.Value()))
} else {
cmd.Println("Missing checksum in object header.")
}
return
}
@ -959,14 +964,27 @@ func saveAndPrintHeader(cmd *cobra.Command, obj *object.Object, filename string)
return printHeader(cmd, obj)
}
func printChecksum(cmd *cobra.Command, name string, recv func() (checksum.Checksum, bool)) {
var strVal string
cs, csSet := recv()
if csSet {
strVal = hex.EncodeToString(cs.Value())
} else {
strVal = "<empty>"
}
cmd.Printf("%s: %s\n", name, strVal)
}
func printHeader(cmd *cobra.Command, obj *object.Object) error {
cmd.Printf("ID: %s\n", obj.ID())
cmd.Printf("CID: %s\n", obj.ContainerID())
cmd.Printf("Owner: %s\n", obj.OwnerID())
cmd.Printf("CreatedAt: %d\n", obj.CreationEpoch())
cmd.Printf("Size: %d\n", obj.PayloadSize())
cmd.Printf("HomoHash: %s\n", hex.EncodeToString(obj.PayloadHomomorphicHash().Sum()))
cmd.Printf("Checksum: %s\n", hex.EncodeToString(obj.PayloadChecksum().Sum()))
printChecksum(cmd, "HomoHash", obj.PayloadHomomorphicHash)
printChecksum(cmd, "Checksum", obj.PayloadChecksum)
cmd.Printf("Type: %s\n", obj.Type())
cmd.Println("Attributes:")

View file

@ -255,7 +255,7 @@ func getSG(cmd *cobra.Command, _ []string) {
cmd.Printf("Expiration epoch: %d\n", sg.ExpirationEpoch())
cmd.Printf("Group size: %d\n", sg.ValidationDataSize())
cmd.Printf("Group hash: %s\n", sg.ValidationDataHash())
printChecksum(cmd, "Group hash", sg.ValidationDataHash)
if members := sg.Members(); len(members) > 0 {
cmd.Println("Members:")

2
go.mod
View file

@ -19,7 +19,7 @@ require (
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220321144137-d5a9af5860af // indirect
github.com/nspcc-dev/neofs-api-go/v2 v2.12.1
github.com/nspcc-dev/neofs-contract v0.14.2
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220413072812-c961aea14446
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220413075357-96892d7bc4a8
github.com/nspcc-dev/tzhash v1.5.2
github.com/panjf2000/ants/v2 v2.4.0
github.com/paulmach/orb v0.2.2

BIN
go.sum

Binary file not shown.

View file

@ -152,11 +152,11 @@ func generateObjectWithCID(t testing.TB, cid *cid.ID) *object.Object {
ver.SetMajor(2)
ver.SetMinor(1)
csum := new(checksum.Checksum)
var csum checksum.Checksum
csum.SetSHA256(sha256.Sum256(owner.PublicKeyToIDBytes(&test.DecodeKey(-1).PublicKey)))
csumTZ := new(checksum.Checksum)
csumTZ.SetTillichZemor(tz.Sum(csum.Sum()))
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := object.New()
obj.SetID(testOID())

View file

@ -72,11 +72,11 @@ func generateObjectWithCID(t testing.TB, cid *cid.ID) *object.Object {
ver.SetMajor(2)
ver.SetMinor(1)
csum := new(checksum.Checksum)
var csum checksum.Checksum
csum.SetSHA256(sha256.Sum256(owner.PublicKeyToIDBytes(&test.DecodeKey(-1).PublicKey)))
csumTZ := new(checksum.Checksum)
csumTZ.SetTillichZemor(tz.Sum(csum.Sum()))
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := object.New()
obj.SetID(testOID())

View file

@ -247,10 +247,12 @@ func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun
addr := object.AddressOf(obj)
objKey := objectKey(addr.ObjectID())
cs, _ := obj.PayloadChecksum()
// index payload hashes
err := f(tx, namedBucketItem{
name: payloadHashBucketName(addr.ContainerID()),
key: obj.PayloadChecksum().Sum(),
key: cs.Value(),
val: objKey,
})
if err != nil {

View file

@ -516,7 +516,8 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr *addressSDK.Address, f object.
case v2object.FilterHeaderVersion:
data = []byte(obj.Version().String())
case v2object.FilterHeaderHomomorphicHash:
data = obj.PayloadHomomorphicHash().Sum()
cs, _ := obj.PayloadHomomorphicHash()
data = cs.Value()
case v2object.FilterHeaderCreationEpoch:
data = make([]byte, 8)
binary.LittleEndian.PutUint64(data, obj.CreationEpoch())

View file

@ -358,16 +358,18 @@ func TestDB_SelectPayloadHash(t *testing.T) {
err = putBig(db, raw2)
require.NoError(t, err)
cs, _ := raw1.PayloadChecksum()
fs := objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderPayloadHash,
hex.EncodeToString(raw1.PayloadChecksum().Sum()),
hex.EncodeToString(cs.Value()),
objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, object.AddressOf(raw1))
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderPayloadHash,
hex.EncodeToString(raw1.PayloadChecksum().Sum()),
hex.EncodeToString(cs.Value()),
objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs, object.AddressOf(raw2))
@ -407,16 +409,18 @@ func TestDB_SelectWithSlowFilters(t *testing.T) {
require.NoError(t, err)
t.Run("object with TZHash", func(t *testing.T) {
cs, _ := raw1.PayloadHomomorphicHash()
fs := objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderHomomorphicHash,
hex.EncodeToString(raw1.PayloadHomomorphicHash().Sum()),
hex.EncodeToString(cs.Value()),
objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, object.AddressOf(raw1))
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderHomomorphicHash,
hex.EncodeToString(raw1.PayloadHomomorphicHash().Sum()),
hex.EncodeToString(cs.Value()),
objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs, object.AddressOf(raw2))

View file

@ -86,11 +86,11 @@ func generateObjectWithPayload(cid *cid.ID, data []byte) *object.Object {
ver.SetMajor(2)
ver.SetMinor(1)
csum := new(checksum.Checksum)
var csum checksum.Checksum
csum.SetSHA256(sha256.Sum256(data))
csumTZ := new(checksum.Checksum)
csumTZ.SetTillichZemor(tz.Sum(csum.Sum()))
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := object.New()
obj.SetID(generateOID())

View file

@ -256,9 +256,11 @@ func (c *Context) updateHeadResponses(hdr *object.Object) {
strID := hdr.ID().String()
cs, _ := hdr.PayloadHomomorphicHash()
if _, ok := c.headResponses[strID]; !ok {
c.headResponses[strID] = shortHeader{
tzhash: hdr.PayloadHomomorphicHash().Sum(),
tzhash: cs.Value(),
objectSize: hdr.PayloadSize(),
}
}

View file

@ -90,13 +90,12 @@ func (c *Context) checkStorageGroupPoR(ind int, sg *oidSDK.ID) {
// update cache for PoR and PDP audit checks
c.updateHeadResponses(hdr)
cs, _ := hdr.PayloadHomomorphicHash()
if len(tzHash) == 0 {
tzHash = hdr.PayloadHomomorphicHash().Sum()
tzHash = cs.Value()
} else {
tzHash, err = tz.Concat([][]byte{
tzHash,
hdr.PayloadHomomorphicHash().Sum(),
})
tzHash, err = tz.Concat([][]byte{tzHash, cs.Value()})
if err != nil {
c.log.Debug("can't concatenate tz hash",
zap.String("oid", members[i].String()),
@ -116,7 +115,8 @@ func (c *Context) checkStorageGroupPoR(ind int, sg *oidSDK.ID) {
c.porRetries.Add(accRetries)
sizeCheck := storageGroup.ValidationDataSize() == totalSize
tzCheck := bytes.Equal(tzHash, storageGroup.ValidationDataHash().Sum())
cs, _ := storageGroup.ValidationDataHash()
tzCheck := bytes.Equal(tzHash, cs.Value())
if sizeCheck && tzCheck {
c.report.PassedPoR(sg) // write report

View file

@ -71,23 +71,29 @@ func headersFromObject(obj *object.Object, addr *objectSDKAddress.Address) []eac
k: acl.FilterObjectVersion,
v: obj.Version().String(),
},
// payload hash
&sysObjHdr{
k: acl.FilterObjectPayloadHash,
v: obj.PayloadChecksum().String(),
},
// object type
&sysObjHdr{
k: acl.FilterObjectType,
v: obj.Type().String(),
},
// payload homomorphic hash
&sysObjHdr{
k: acl.FilterObjectHomomorphicHash,
v: obj.PayloadHomomorphicHash().String(),
},
)
cs, ok := obj.PayloadChecksum()
if ok {
res = append(res, &sysObjHdr{
k: acl.FilterObjectPayloadHash,
v: cs.String(),
})
}
cs, ok = obj.PayloadHomomorphicHash()
if ok {
res = append(res, &sysObjHdr{
k: acl.FilterObjectHomomorphicHash,
v: cs.String(),
})
}
attrs := obj.Attributes()
for i := range attrs {
res = append(res, &attrs[i]) // only pointer attrs can implement eaclSDK.Header interface

View file

@ -54,7 +54,11 @@ func (t *validatingTarget) WriteHeader(obj *objectSDK.Object) error {
return ErrExceedingMaxSize
}
cs := obj.PayloadChecksum()
cs, csSet := obj.PayloadChecksum()
if !csSet {
return errors.New("missing payload checksum")
}
switch typ := cs.Type(); typ {
default:
return fmt.Errorf("(%T) unsupported payload checksum type %v", t, typ)
@ -64,7 +68,7 @@ func (t *validatingTarget) WriteHeader(obj *objectSDK.Object) error {
t.hash = tz.New()
}
t.checksum = cs.Sum()
t.checksum = cs.Value()
}
if err := t.fmt.Validate(obj, t.unpreparedObject); err != nil {

View file

@ -32,7 +32,8 @@ func CollectMembers(r objutil.HeadReceiver, cid *cid.ID, members []oidSDK.ID) (*
if err := objutil.IterateAllSplitLeaves(r, addr, func(leaf *object.Object) {
phyMembers = append(phyMembers, *leaf.ID())
sumPhySize += leaf.PayloadSize()
phyHashes = append(phyHashes, leaf.PayloadHomomorphicHash().Sum())
cs, _ := leaf.PayloadHomomorphicHash()
phyHashes = append(phyHashes, cs.Value())
}); err != nil {
return nil, err
}
@ -43,7 +44,7 @@ func CollectMembers(r objutil.HeadReceiver, cid *cid.ID, members []oidSDK.ID) (*
return nil, err
}
cs := checksum.New()
var cs checksum.Checksum
tzHash := [64]byte{}
copy(tzHash[:], sumHash)
cs.SetTillichZemor(tzHash)

View file

@ -128,34 +128,34 @@ func payloadHashersForObject(obj *object.Object) []*payloadChecksumHasher {
return []*payloadChecksumHasher{
{
hasher: sha256.New(),
checksumWriter: func(cs []byte) {
if ln := len(cs); ln != sha256.Size {
checksumWriter: func(binChecksum []byte) {
if ln := len(binChecksum); ln != sha256.Size {
panic(fmt.Sprintf("wrong checksum length: expected %d, has %d", sha256.Size, ln))
}
csSHA := [sha256.Size]byte{}
copy(csSHA[:], cs)
copy(csSHA[:], binChecksum)
checksum := checksum.New()
checksum.SetSHA256(csSHA)
var cs checksum.Checksum
cs.SetSHA256(csSHA)
obj.SetPayloadChecksum(checksum)
obj.SetPayloadChecksum(cs)
},
},
{
hasher: tz.New(),
checksumWriter: func(cs []byte) {
if ln := len(cs); ln != tz.Size {
checksumWriter: func(binChecksum []byte) {
if ln := len(binChecksum); ln != tz.Size {
panic(fmt.Sprintf("wrong checksum length: expected %d, has %d", tz.Size, ln))
}
csTZ := [tz.Size]byte{}
copy(csTZ[:], cs)
copy(csTZ[:], binChecksum)
checksum := checksum.New()
checksum.SetTillichZemor(csTZ)
var cs checksum.Checksum
cs.SetTillichZemor(csTZ)
obj.SetPayloadHomomorphicHash(checksum)
obj.SetPayloadHomomorphicHash(cs)
},
},
}