[#30] object_manager: Implement object transformers

Define object writer interface. Implement payload size limiter that
restricts payload size of the object. Implement object format moulder that
fill verification fields and finalizes object structure.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78
 	github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200916115135-ff325b877023
 	github.com/nspcc-dev/neofs-crypto v0.3.0
+	github.com/nspcc-dev/tzhash v1.4.0
 	github.com/panjf2000/ants/v2 v2.3.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.6.0

go.sum

@@ -267,10 +267,6 @@ github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:
 github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=
 github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78 h1:stIa+nBXK8uDY/JZaxIZzAUfkzfaotVw2FbnHxO4aZI=
 github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=
-github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200902121740-5a6dff8c83ba h1:S8YfqJ2F8o4KY5dP7DcBNXdJsWtTK/vtD9mcF4J/g6Y=
-github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200902121740-5a6dff8c83ba/go.mod h1:f5Z4UggDgVgZXnDNr2ItdGwBpVYJBKlm5Yyu2XYEiAc=
-github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200911095622-47fd771ee4c5 h1:+/VTNh3tHfuBHF+Xyf/QVO/U7QTcWyJ0rPHl+b8i0kU=
-github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200911095622-47fd771ee4c5/go.mod h1:FsFd1z4YzoEgPlltsUgnqna9qhcF87RHYjot0pby2L4=
 github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200916115135-ff325b877023 h1:tltnqudivH6TBzs4DEouLx9rwPUBuvn7bjm4EZyosUc=
 github.com/nspcc-dev/neofs-api-go v1.3.1-0.20200916115135-ff325b877023/go.mod h1:FsFd1z4YzoEgPlltsUgnqna9qhcF87RHYjot0pby2L4=
 github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA=
@@ -280,6 +276,8 @@ github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BE
 github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
 github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE=
 github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
+github.com/nspcc-dev/tzhash v1.4.0 h1:RVIR+mxOBHl58CE99+DXtE31ylD5PEkZSoWqoj4fVjg=
+github.com/nspcc-dev/tzhash v1.4.0/go.mod h1:Z8gp/VZbyWgPhaMp/KTmeoW5UTynp/N60g0jTtSzBws=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -488,6 +486,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=

pkg/services/object_manager/transformer/fmt.go

@@ -0,0 +1,97 @@
+package transformer
+
+import (
+	"crypto/ecdsa"
+
+	"github.com/nspcc-dev/neofs-api-go/pkg"
+	objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
+	"github.com/nspcc-dev/neofs-node/pkg/core/object"
+	"github.com/pkg/errors"
+)
+
+type formatter struct {
+	nextTarget ObjectTarget
+
+	key *ecdsa.PrivateKey
+
+	obj *object.RawObject
+
+	sz uint64
+}
+
+// NewFormatTarget returns ObjectTarget instance that finalizes object structure
+// and writes it to the next target.
+//
+// Chunks must be written before the WriteHeader call.
+//
+// Object changes:
+// - sets version to current SDK version;
+// - sets payload size to the total length of all written chunks;
+// - calculates and sets verification fields (ID, Signature).
+func NewFormatTarget(key *ecdsa.PrivateKey, nextTarget ObjectTarget) ObjectTarget {
+	return &formatter{
+		nextTarget: nextTarget,
+		key:        key,
+	}
+}
+
+func (f *formatter) WriteHeader(obj *object.RawObject) error {
+	f.obj = obj
+
+	return nil
+}
+
+func (f *formatter) Write(p []byte) (n int, err error) {
+	n, err = f.nextTarget.Write(p)
+
+	f.sz += uint64(n)
+
+	return
+}
+
+func (f *formatter) Close() (*AccessIdentifiers, error) {
+	f.obj.SetVersion(pkg.SDKVersion())
+	f.obj.SetPayloadSize(f.sz)
+
+	var parID *objectSDK.ID
+
+	if par := f.obj.GetParent(); par != nil && par.ToV2().GetHeader() != nil {
+		rawPar := objectSDK.NewRawFromV2(par.ToV2())
+
+		if err := setIDAndSignature(f.key, rawPar); err != nil {
+			return nil, errors.Wrap(err, "could not finalize parent object")
+		}
+
+		parID = rawPar.GetID()
+
+		f.obj.SetParent(rawPar.Object())
+	}
+
+	if err := setIDAndSignature(f.key, f.obj.SDK()); err != nil {
+		return nil, errors.Wrap(err, "could not finalize object")
+	}
+
+	if err := f.nextTarget.WriteHeader(f.obj); err != nil {
+		return nil, errors.Wrap(err, "could not write header to next target")
+	}
+
+	if _, err := f.nextTarget.Close(); err != nil {
+		return nil, errors.Wrap(err, "could not close next target")
+	}
+
+	return new(AccessIdentifiers).
+		WithSelfID(f.obj.GetID()).
+		WithParentID(parID), nil
+}
+
+func setIDAndSignature(key *ecdsa.PrivateKey, obj *objectSDK.RawObject) error {
+	if err := objectSDK.CalculateAndSetID(obj); err != nil {
+		return errors.Wrap(err, "could not set identifier")
+	}
+
+	if err := objectSDK.CalculateAndSetSignature(key, obj); err != nil {
+		return errors.Wrap(err, "could not set signature")
+	}
+
+	return nil
+}

pkg/services/object_manager/transformer/transformer.go

@@ -0,0 +1,252 @@
+package transformer
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"hash"
+	"io"
+
+	"github.com/nspcc-dev/neofs-api-go/pkg"
+	objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
+	"github.com/nspcc-dev/neofs-node/pkg/core/object"
+	"github.com/nspcc-dev/tzhash/tz"
+	"github.com/pkg/errors"
+)
+
+type payloadSizeLimiter struct {
+	maxSize, written uint64
+
+	targetInit func() ObjectTarget
+
+	target ObjectTarget
+
+	current, parent *object.RawObject
+
+	currentHashers, parentHashers []*payloadChecksumHasher
+
+	previous []*objectSDK.ID
+
+	chunkWriter io.Writer
+}
+
+type payloadChecksumHasher struct {
+	hasher hash.Hash
+
+	checksumWriter func([]byte)
+}
+
+const tzChecksumSize = 64
+
+// NewPayloadSizeLimiter returns ObjectTarget instance that restricts payload length
+// of the writing object and writes generated objects to targets from initializer.
+//
+// Objects w/ payload size less or equal than max size remain untouched.
+//
+// TODO: describe behavior in details.
+func NewPayloadSizeLimiter(maxSize uint64, targetInit TargetInitializer) ObjectTarget {
+	return &payloadSizeLimiter{
+		maxSize:    maxSize,
+		targetInit: targetInit,
+	}
+}
+
+func (s *payloadSizeLimiter) WriteHeader(hdr *object.RawObject) error {
+	s.current = fromObject(hdr)
+
+	return nil
+}
+
+func (s *payloadSizeLimiter) Write(p []byte) (int, error) {
+	if err := s.writeChunk(p); err != nil {
+		return 0, err
+	}
+
+	return len(p), nil
+}
+
+func (s *payloadSizeLimiter) Close() (*AccessIdentifiers, error) {
+	return s.release(true)
+}
+
+func (s *payloadSizeLimiter) initialize() {
+	// if it is an object after the 1st
+	if ln := len(s.previous); ln > 0 {
+		// initialize parent object once (after 1st object)
+		if ln == 1 {
+			s.parent = s.current
+			s.parentHashers = s.currentHashers
+			s.current = fromObject(s.parent)
+		}
+
+		// set previous object to the last previous identifier
+		s.current.SetPreviousID(s.previous[ln-1])
+	}
+
+	s.initializeCurrent()
+}
+
+func fromObject(obj *object.RawObject) *object.RawObject {
+	res := object.NewRaw()
+	res.SetContainerID(obj.GetContainerID())
+	res.SetOwnerID(obj.GetOwnerID())
+	res.SetAttributes(obj.GetAttributes()...)
+
+	return res
+}
+
+func (s *payloadSizeLimiter) initializeCurrent() {
+	// initialize current object target
+	s.target = s.targetInit()
+
+	// create payload hashers
+	s.currentHashers = []*payloadChecksumHasher{
+		{
+			hasher: sha256.New(),
+			checksumWriter: func(cs []byte) {
+				if ln := len(cs); ln != sha256.Size {
+					panic(fmt.Sprintf("wrong checksum length: expected %d, has %d", ln, sha256.Size))
+				}
+
+				csSHA := [sha256.Size]byte{}
+				copy(csSHA[:], cs)
+
+				checksum := pkg.NewChecksum()
+				checksum.SetSHA256(csSHA)
+
+				s.current.SetPayloadChecksum(checksum)
+			},
+		},
+		{
+			hasher: tz.New(),
+			checksumWriter: func(cs []byte) {
+				if ln := len(cs); ln != tzChecksumSize {
+					panic(fmt.Sprintf("wrong checksum length: expected %d, has %d", ln, tzChecksumSize))
+				}
+
+				csTZ := [tzChecksumSize]byte{}
+				copy(csTZ[:], cs)
+
+				checksum := pkg.NewChecksum()
+				checksum.SetTillichZemor(csTZ)
+
+				s.current.SetPayloadHomomorphicHash(checksum)
+			},
+		},
+	}
+
+	// compose multi-writer from target and all payload hashers
+	ws := make([]io.Writer, 0, 1+len(s.currentHashers)+len(s.parentHashers))
+
+	ws = append(ws, s.target)
+
+	for i := range s.currentHashers {
+		ws = append(ws, s.currentHashers[i].hasher)
+	}
+
+	for i := range s.parentHashers {
+		ws = append(ws, s.parentHashers[i].hasher)
+	}
+
+	s.chunkWriter = io.MultiWriter(ws...)
+}
+
+func (s *payloadSizeLimiter) release(close bool) (*AccessIdentifiers, error) {
+	// Arg close is true only from Close method.
+	// We finalize parent and generate linking objects only if it is more
+	// than 1 object in split-chain.
+	withParent := close && len(s.previous) > 0
+
+	if withParent {
+		writeHashes(s.parentHashers)
+		s.parent.SetPayloadSize(s.written)
+		s.current.SetParent(s.parent.SDK().Object())
+	}
+
+	// release current object
+	writeHashes(s.currentHashers)
+
+	// release current, get its id
+	if err := s.target.WriteHeader(s.current); err != nil {
+		return nil, errors.Wrap(err, "could not write header")
+	}
+
+	ids, err := s.target.Close()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not close target")
+	}
+
+	// save identifier of the released object
+	s.previous = append(s.previous, ids.SelfID())
+
+	if withParent {
+		// generate and release linking object
+		s.initializeLinking()
+		s.initializeCurrent()
+
+		if _, err := s.release(false); err != nil {
+			return nil, errors.Wrap(err, "could not release linking object")
+		}
+	}
+
+	return ids, nil
+}
+
+func writeHashes(hashers []*payloadChecksumHasher) {
+	for i := range hashers {
+		hashers[i].checksumWriter(hashers[i].hasher.Sum(nil))
+
+	}
+}
+
+func (s *payloadSizeLimiter) initializeLinking() {
+	id := s.current.GetParent().GetID()
+	par := objectSDK.NewRaw()
+	par.SetID(id)
+
+	s.current = fromObject(s.current)
+	s.current.SetChildren(s.previous...)
+
+	s.current.SetParent(par.Object())
+}
+
+func (s *payloadSizeLimiter) writeChunk(chunk []byte) error {
+	// statement is true if:
+	//   1. the method is called for the first time;
+	//   2. the previous write of bytes reached exactly the boundary.
+	if s.written%s.maxSize == 0 {
+		// if 2. we need to release current object
+		if s.written > 0 {
+			if _, err := s.release(false); err != nil {
+				return errors.Wrap(err, "could not release object")
+			}
+		}
+
+		// initialize another object
+		s.initialize()
+	}
+
+	var (
+		ln         = uint64(len(chunk))
+		cut        = ln
+		leftToEdge = s.maxSize - s.written%s.maxSize
+	)
+
+	// write bytes no further than the boundary of the current object
+	if ln > leftToEdge {
+		cut = leftToEdge
+	}
+
+	if _, err := s.chunkWriter.Write(chunk[:cut]); err != nil {
+		return errors.Wrap(err, "could not write chunk to target")
+	}
+
+	// increase written bytes counter
+	s.written += cut
+
+	// if there are more bytes in buffer we call method again to start filling another object
+	if ln > leftToEdge {
+		return s.writeChunk(chunk[cut:])
+	}
+
+	return nil
+}

pkg/services/object_manager/transformer/types.go

@@ -0,0 +1,90 @@
+package transformer
+
+import (
+	"io"
+
+	objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
+	"github.com/nspcc-dev/neofs-node/pkg/core/object"
+)
+
+// AccessIdentifiers represents group of the object identifiers
+// that are returned after writing the object.
+// Consists of the ID of the stored object and the ID of the parent object.
+type AccessIdentifiers struct {
+	par, self *objectSDK.ID
+}
+
+// ObjectTarget is an interface of the object writer.
+type ObjectTarget interface {
+	// WriteHeader writes object header w/ payload part.
+	// The payload of the object may be incomplete.
+	//
+	// Must be called exactly once. Control remains with the caller.
+	// Missing a call or re-calling can lead to undefined behavior
+	// that depends on the implementation.
+	//
+	// Must not be called after Close call.
+	WriteHeader(*object.RawObject) error
+
+	// Write writes object payload chunk.
+	//
+	// Can be called multiple times.
+	//
+	// Must not be called after Close call.
+	io.Writer
+
+	// Close is used to finish object writing.
+	//
+	// Close must return access identifiers of the object
+	// that has been written.
+	//
+	// Must be called no more than once. Control remains with the caller.
+	// Re-calling can lead to undefined behavior
+	// that depends on the implementation.
+	Close() (*AccessIdentifiers, error)
+}
+
+// TargetInitializer represents ObjectTarget constructor.
+type TargetInitializer func() ObjectTarget
+
+// SelfID returns identifier of the written object.
+func (a *AccessIdentifiers) SelfID() *objectSDK.ID {
+	if a != nil {
+		return a.self
+	}
+
+	return nil
+}
+
+// WithSelfID returns AccessIdentifiers with passed self identifier.
+func (a *AccessIdentifiers) WithSelfID(v *objectSDK.ID) *AccessIdentifiers {
+	res := a
+	if res == nil {
+		res = new(AccessIdentifiers)
+	}
+
+	res.self = v
+
+	return res
+}
+
+// ParentID return identifier of the parent of the written object.
+func (a *AccessIdentifiers) ParentID() *objectSDK.ID {
+	if a != nil {
+		return a.par
+	}
+
+	return nil
+}
+
+// WithParentID returns AccessIdentifiers with passed parent identifier.
+func (a *AccessIdentifiers) WithParentID(v *objectSDK.ID) *AccessIdentifiers {
+	res := a
+	if res == nil {
+		res = new(AccessIdentifiers)
+	}
+
+	res.par = v
+
+	return res
+}