[#30] object_manager: Implement object transformers

Define object writer interface. Implement payload size limiter that
restricts payload size of the object. Implement object format moulder that
fill verification fields and finalizes object structure.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2020-09-16 17:46:31 +03:00 committed by Alex Vanin
parent 0a6130a924
commit 121137c62b
6 changed files with 443 additions and 4 deletions

1
go.mod
View file

@ -15,6 +15,7 @@ require (
github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78 github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200916115135-ff325b877023 github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200916115135-ff325b877023
github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/nspcc-dev/neofs-crypto v0.3.0
github.com/nspcc-dev/tzhash v1.4.0
github.com/panjf2000/ants/v2 v2.3.0 github.com/panjf2000/ants/v2 v2.3.0
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.6.0 github.com/prometheus/client_golang v1.6.0

7
go.sum
View file

@ -267,10 +267,6 @@ github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:
github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=
github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78 h1:stIa+nBXK8uDY/JZaxIZzAUfkzfaotVw2FbnHxO4aZI= github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78 h1:stIa+nBXK8uDY/JZaxIZzAUfkzfaotVw2FbnHxO4aZI=
github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200902121740-5a6dff8c83ba h1:S8YfqJ2F8o4KY5dP7DcBNXdJsWtTK/vtD9mcF4J/g6Y=
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200902121740-5a6dff8c83ba/go.mod h1:f5Z4UggDgVgZXnDNr2ItdGwBpVYJBKlm5Yyu2XYEiAc=
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200911095622-47fd771ee4c5 h1:+/VTNh3tHfuBHF+Xyf/QVO/U7QTcWyJ0rPHl+b8i0kU=
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200911095622-47fd771ee4c5/go.mod h1:FsFd1z4YzoEgPlltsUgnqna9qhcF87RHYjot0pby2L4=
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200916115135-ff325b877023 h1:tltnqudivH6TBzs4DEouLx9rwPUBuvn7bjm4EZyosUc= github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200916115135-ff325b877023 h1:tltnqudivH6TBzs4DEouLx9rwPUBuvn7bjm4EZyosUc=
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200916115135-ff325b877023/go.mod h1:FsFd1z4YzoEgPlltsUgnqna9qhcF87RHYjot0pby2L4= github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200916115135-ff325b877023/go.mod h1:FsFd1z4YzoEgPlltsUgnqna9qhcF87RHYjot0pby2L4=
github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA=
@ -280,6 +276,8 @@ github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BE
github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE=
github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
github.com/nspcc-dev/tzhash v1.4.0 h1:RVIR+mxOBHl58CE99+DXtE31ylD5PEkZSoWqoj4fVjg=
github.com/nspcc-dev/tzhash v1.4.0/go.mod h1:Z8gp/VZbyWgPhaMp/KTmeoW5UTynp/N60g0jTtSzBws=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@ -488,6 +486,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=

View file

@ -0,0 +1,97 @@
package transformer
import (
"crypto/ecdsa"
"github.com/nspcc-dev/neofs-api-go/pkg"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/pkg/errors"
)
type formatter struct {
nextTarget ObjectTarget
key *ecdsa.PrivateKey
obj *object.RawObject
sz uint64
}
// NewFormatTarget returns ObjectTarget instance that finalizes object structure
// and writes it to the next target.
//
// Chunks must be written before the WriteHeader call.
//
// Object changes:
// - sets version to current SDK version;
// - sets payload size to the total length of all written chunks;
// - calculates and sets verification fields (ID, Signature).
func NewFormatTarget(key *ecdsa.PrivateKey, nextTarget ObjectTarget) ObjectTarget {
return &formatter{
nextTarget: nextTarget,
key: key,
}
}
func (f *formatter) WriteHeader(obj *object.RawObject) error {
f.obj = obj
return nil
}
func (f *formatter) Write(p []byte) (n int, err error) {
n, err = f.nextTarget.Write(p)
f.sz += uint64(n)
return
}
func (f *formatter) Close() (*AccessIdentifiers, error) {
f.obj.SetVersion(pkg.SDKVersion())
f.obj.SetPayloadSize(f.sz)
var parID *objectSDK.ID
if par := f.obj.GetParent(); par != nil && par.ToV2().GetHeader() != nil {
rawPar := objectSDK.NewRawFromV2(par.ToV2())
if err := setIDAndSignature(f.key, rawPar); err != nil {
return nil, errors.Wrap(err, "could not finalize parent object")
}
parID = rawPar.GetID()
f.obj.SetParent(rawPar.Object())
}
if err := setIDAndSignature(f.key, f.obj.SDK()); err != nil {
return nil, errors.Wrap(err, "could not finalize object")
}
if err := f.nextTarget.WriteHeader(f.obj); err != nil {
return nil, errors.Wrap(err, "could not write header to next target")
}
if _, err := f.nextTarget.Close(); err != nil {
return nil, errors.Wrap(err, "could not close next target")
}
return new(AccessIdentifiers).
WithSelfID(f.obj.GetID()).
WithParentID(parID), nil
}
func setIDAndSignature(key *ecdsa.PrivateKey, obj *objectSDK.RawObject) error {
if err := objectSDK.CalculateAndSetID(obj); err != nil {
return errors.Wrap(err, "could not set identifier")
}
if err := objectSDK.CalculateAndSetSignature(key, obj); err != nil {
return errors.Wrap(err, "could not set signature")
}
return nil
}

View file

@ -0,0 +1,252 @@
package transformer
import (
"crypto/sha256"
"fmt"
"hash"
"io"
"github.com/nspcc-dev/neofs-api-go/pkg"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/tzhash/tz"
"github.com/pkg/errors"
)
type payloadSizeLimiter struct {
maxSize, written uint64
targetInit func() ObjectTarget
target ObjectTarget
current, parent *object.RawObject
currentHashers, parentHashers []*payloadChecksumHasher
previous []*objectSDK.ID
chunkWriter io.Writer
}
type payloadChecksumHasher struct {
hasher hash.Hash
checksumWriter func([]byte)
}
const tzChecksumSize = 64
// NewPayloadSizeLimiter returns ObjectTarget instance that restricts payload length
// of the writing object and writes generated objects to targets from initializer.
//
// Objects w/ payload size less or equal than max size remain untouched.
//
// TODO: describe behavior in details.
func NewPayloadSizeLimiter(maxSize uint64, targetInit TargetInitializer) ObjectTarget {
return &payloadSizeLimiter{
maxSize: maxSize,
targetInit: targetInit,
}
}
func (s *payloadSizeLimiter) WriteHeader(hdr *object.RawObject) error {
s.current = fromObject(hdr)
return nil
}
func (s *payloadSizeLimiter) Write(p []byte) (int, error) {
if err := s.writeChunk(p); err != nil {
return 0, err
}
return len(p), nil
}
func (s *payloadSizeLimiter) Close() (*AccessIdentifiers, error) {
return s.release(true)
}
func (s *payloadSizeLimiter) initialize() {
// if it is an object after the 1st
if ln := len(s.previous); ln > 0 {
// initialize parent object once (after 1st object)
if ln == 1 {
s.parent = s.current
s.parentHashers = s.currentHashers
s.current = fromObject(s.parent)
}
// set previous object to the last previous identifier
s.current.SetPreviousID(s.previous[ln-1])
}
s.initializeCurrent()
}
func fromObject(obj *object.RawObject) *object.RawObject {
res := object.NewRaw()
res.SetContainerID(obj.GetContainerID())
res.SetOwnerID(obj.GetOwnerID())
res.SetAttributes(obj.GetAttributes()...)
return res
}
func (s *payloadSizeLimiter) initializeCurrent() {
// initialize current object target
s.target = s.targetInit()
// create payload hashers
s.currentHashers = []*payloadChecksumHasher{
{
hasher: sha256.New(),
checksumWriter: func(cs []byte) {
if ln := len(cs); ln != sha256.Size {
panic(fmt.Sprintf("wrong checksum length: expected %d, has %d", ln, sha256.Size))
}
csSHA := [sha256.Size]byte{}
copy(csSHA[:], cs)
checksum := pkg.NewChecksum()
checksum.SetSHA256(csSHA)
s.current.SetPayloadChecksum(checksum)
},
},
{
hasher: tz.New(),
checksumWriter: func(cs []byte) {
if ln := len(cs); ln != tzChecksumSize {
panic(fmt.Sprintf("wrong checksum length: expected %d, has %d", ln, tzChecksumSize))
}
csTZ := [tzChecksumSize]byte{}
copy(csTZ[:], cs)
checksum := pkg.NewChecksum()
checksum.SetTillichZemor(csTZ)
s.current.SetPayloadHomomorphicHash(checksum)
},
},
}
// compose multi-writer from target and all payload hashers
ws := make([]io.Writer, 0, 1+len(s.currentHashers)+len(s.parentHashers))
ws = append(ws, s.target)
for i := range s.currentHashers {
ws = append(ws, s.currentHashers[i].hasher)
}
for i := range s.parentHashers {
ws = append(ws, s.parentHashers[i].hasher)
}
s.chunkWriter = io.MultiWriter(ws...)
}
func (s *payloadSizeLimiter) release(close bool) (*AccessIdentifiers, error) {
// Arg close is true only from Close method.
// We finalize parent and generate linking objects only if it is more
// than 1 object in split-chain.
withParent := close && len(s.previous) > 0
if withParent {
writeHashes(s.parentHashers)
s.parent.SetPayloadSize(s.written)
s.current.SetParent(s.parent.SDK().Object())
}
// release current object
writeHashes(s.currentHashers)
// release current, get its id
if err := s.target.WriteHeader(s.current); err != nil {
return nil, errors.Wrap(err, "could not write header")
}
ids, err := s.target.Close()
if err != nil {
return nil, errors.Wrap(err, "could not close target")
}
// save identifier of the released object
s.previous = append(s.previous, ids.SelfID())
if withParent {
// generate and release linking object
s.initializeLinking()
s.initializeCurrent()
if _, err := s.release(false); err != nil {
return nil, errors.Wrap(err, "could not release linking object")
}
}
return ids, nil
}
func writeHashes(hashers []*payloadChecksumHasher) {
for i := range hashers {
hashers[i].checksumWriter(hashers[i].hasher.Sum(nil))
}
}
func (s *payloadSizeLimiter) initializeLinking() {
id := s.current.GetParent().GetID()
par := objectSDK.NewRaw()
par.SetID(id)
s.current = fromObject(s.current)
s.current.SetChildren(s.previous...)
s.current.SetParent(par.Object())
}
func (s *payloadSizeLimiter) writeChunk(chunk []byte) error {
// statement is true if:
// 1. the method is called for the first time;
// 2. the previous write of bytes reached exactly the boundary.
if s.written%s.maxSize == 0 {
// if 2. we need to release current object
if s.written > 0 {
if _, err := s.release(false); err != nil {
return errors.Wrap(err, "could not release object")
}
}
// initialize another object
s.initialize()
}
var (
ln = uint64(len(chunk))
cut = ln
leftToEdge = s.maxSize - s.written%s.maxSize
)
// write bytes no further than the boundary of the current object
if ln > leftToEdge {
cut = leftToEdge
}
if _, err := s.chunkWriter.Write(chunk[:cut]); err != nil {
return errors.Wrap(err, "could not write chunk to target")
}
// increase written bytes counter
s.written += cut
// if there are more bytes in buffer we call method again to start filling another object
if ln > leftToEdge {
return s.writeChunk(chunk[cut:])
}
return nil
}

View file

@ -0,0 +1,90 @@
package transformer
import (
"io"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
)
// AccessIdentifiers represents group of the object identifiers
// that are returned after writing the object.
// Consists of the ID of the stored object and the ID of the parent object.
type AccessIdentifiers struct {
par, self *objectSDK.ID
}
// ObjectTarget is an interface of the object writer.
type ObjectTarget interface {
// WriteHeader writes object header w/ payload part.
// The payload of the object may be incomplete.
//
// Must be called exactly once. Control remains with the caller.
// Missing a call or re-calling can lead to undefined behavior
// that depends on the implementation.
//
// Must not be called after Close call.
WriteHeader(*object.RawObject) error
// Write writes object payload chunk.
//
// Can be called multiple times.
//
// Must not be called after Close call.
io.Writer
// Close is used to finish object writing.
//
// Close must return access identifiers of the object
// that has been written.
//
// Must be called no more than once. Control remains with the caller.
// Re-calling can lead to undefined behavior
// that depends on the implementation.
Close() (*AccessIdentifiers, error)
}
// TargetInitializer represents ObjectTarget constructor.
type TargetInitializer func() ObjectTarget
// SelfID returns identifier of the written object.
func (a *AccessIdentifiers) SelfID() *objectSDK.ID {
if a != nil {
return a.self
}
return nil
}
// WithSelfID returns AccessIdentifiers with passed self identifier.
func (a *AccessIdentifiers) WithSelfID(v *objectSDK.ID) *AccessIdentifiers {
res := a
if res == nil {
res = new(AccessIdentifiers)
}
res.self = v
return res
}
// ParentID return identifier of the parent of the written object.
func (a *AccessIdentifiers) ParentID() *objectSDK.ID {
if a != nil {
return a.par
}
return nil
}
// WithParentID returns AccessIdentifiers with passed parent identifier.
func (a *AccessIdentifiers) WithParentID(v *objectSDK.ID) *AccessIdentifiers {
res := a
if res == nil {
res = new(AccessIdentifiers)
}
res.par = v
return res
}