Initial commit

Initial public review release v0.10.0
This commit is contained in:
alexvanin 2020-07-10 17:17:51 +03:00 committed by Stanislav Bogatyrev
commit dadfd90dcd
276 changed files with 46331 additions and 0 deletions

0
lib/.gitkeep Normal file
View file

94
lib/acl/action.go Normal file
View file

@ -0,0 +1,94 @@
package acl
import (
"bytes"
"github.com/nspcc-dev/neofs-api-go/acl"
)
// RequestInfo is an interface of request information needed for extended ACL check.
type RequestInfo interface {
TypedHeaderSource
// Must return the binary representation of request initiator's key.
Key() []byte
// Must return true if request corresponds to operation type.
TypeOf(acl.OperationType) bool
// Must return true if request has passed target.
TargetOf(acl.Target) bool
}
// ExtendedACLChecker is an interface of extended ACL checking tool.
type ExtendedACLChecker interface {
// Must return an action according to the results of applying the ACL table rules to request.
//
// Must return ActionUndefined if it is unable to explicitly calculate the action.
Action(acl.ExtendedACLTable, RequestInfo) acl.ExtendedACLAction
}
type extendedACLChecker struct{}
// NewExtendedACLChecker creates a new extended ACL checking tool and returns ExtendedACLChecker interface.
func NewExtendedACLChecker() ExtendedACLChecker {
return new(extendedACLChecker)
}
// Action returns an action for passed request based on information about it and ACL table.
//
// Returns action of the first suitable table record, or ActionUndefined in the absence thereof.
//
// If passed ExtendedACLTable is nil, ActionUndefined returns.
// If passed RequestInfo is nil, ActionUndefined returns.
func (s extendedACLChecker) Action(table acl.ExtendedACLTable, req RequestInfo) acl.ExtendedACLAction {
if table == nil {
return acl.ActionUndefined
} else if req == nil {
return acl.ActionUndefined
}
for _, record := range table.Records() {
// check type of operation
if !req.TypeOf(record.OperationType()) {
continue
}
// check target
if !targetMatches(req, record.TargetList()) {
continue
}
// check headers
switch MatchFilters(req, record.HeaderFilters()) {
case mResUndefined:
// headers of some type could not be composed => allow
return acl.ActionAllow
case mResMatch:
return record.Action()
}
}
return acl.ActionAllow
}
// returns true if one of ExtendedACLTarget has suitable target OR suitable public key.
func targetMatches(req RequestInfo, list []acl.ExtendedACLTarget) bool {
rKey := req.Key()
for _, target := range list {
// check public key match
for _, key := range target.KeyList() {
if bytes.Equal(key, rKey) {
return true
}
}
// check target group match
if req.TargetOf(target.Target()) {
return true
}
}
return false
}

163
lib/acl/action_test.go Normal file
View file

@ -0,0 +1,163 @@
package acl
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/stretchr/testify/require"
)
type testExtendedACLTable struct {
records []acl.ExtendedACLRecord
}
type testRequestInfo struct {
headers []acl.TypedHeader
key []byte
opType acl.OperationType
target acl.Target
}
type testEACLRecord struct {
opType acl.OperationType
filters []acl.HeaderFilter
targets []acl.ExtendedACLTarget
action acl.ExtendedACLAction
}
type testEACLTarget struct {
target acl.Target
keys [][]byte
}
func (s testEACLTarget) Target() acl.Target {
return s.target
}
func (s testEACLTarget) KeyList() [][]byte {
return s.keys
}
func (s testEACLRecord) OperationType() acl.OperationType {
return s.opType
}
func (s testEACLRecord) HeaderFilters() []acl.HeaderFilter {
return s.filters
}
func (s testEACLRecord) TargetList() []acl.ExtendedACLTarget {
return s.targets
}
func (s testEACLRecord) Action() acl.ExtendedACLAction {
return s.action
}
func (s testRequestInfo) HeadersOfType(typ acl.HeaderType) ([]acl.Header, bool) {
res := make([]acl.Header, 0, len(s.headers))
for i := range s.headers {
if s.headers[i].HeaderType() == typ {
res = append(res, s.headers[i])
}
}
return res, true
}
func (s testRequestInfo) Key() []byte {
return s.key
}
func (s testRequestInfo) TypeOf(t acl.OperationType) bool {
return s.opType == t
}
func (s testRequestInfo) TargetOf(t acl.Target) bool {
return s.target == t
}
func (s testExtendedACLTable) Records() []acl.ExtendedACLRecord {
return s.records
}
func TestExtendedACLChecker_Action(t *testing.T) {
s := NewExtendedACLChecker()
// nil ExtendedACLTable
require.Equal(t, acl.ActionUndefined, s.Action(nil, nil))
// create test ExtendedACLTable
table := new(testExtendedACLTable)
// nil RequestInfo
require.Equal(t, acl.ActionUndefined, s.Action(table, nil))
// create test RequestInfo
req := new(testRequestInfo)
// create test ExtendedACLRecord
record := new(testEACLRecord)
table.records = append(table.records, record)
// set different OperationType
record.opType = acl.OperationType(3)
req.opType = record.opType + 1
require.Equal(t, acl.ActionAllow, s.Action(table, req))
// set equal OperationType
req.opType = record.opType
// create test ExtendedACLTarget through group
target := new(testEACLTarget)
record.targets = append(record.targets, target)
// set not matching ExtendedACLTarget
target.target = acl.Target(5)
req.target = target.target + 1
require.Equal(t, acl.ActionAllow, s.Action(table, req))
// set matching ExtendedACLTarget
req.target = target.target
// create test HeaderFilter
fHeader := new(testTypedHeader)
hFilter := &testHeaderFilter{
TypedHeader: fHeader,
}
record.filters = append(record.filters, hFilter)
// create test TypedHeader
header := new(testTypedHeader)
req.headers = append(req.headers, header)
// set not matching values
header.t = hFilter.HeaderType() + 1
require.Equal(t, acl.ActionAllow, s.Action(table, req))
// set matching values
header.k = "key"
header.v = "value"
fHeader.t = header.HeaderType()
fHeader.k = header.Name()
fHeader.v = header.Value()
hFilter.t = acl.StringEqual
// set ExtendedACLAction
record.action = acl.ExtendedACLAction(7)
require.Equal(t, record.action, s.Action(table, req))
// set matching ExtendedACLTarget through key
target.target = req.target + 1
req.key = []byte{1, 2, 3}
target.keys = append(target.keys, req.key)
require.Equal(t, record.action, s.Action(table, req))
}

179
lib/acl/basic.go Normal file
View file

@ -0,0 +1,179 @@
package acl
import (
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-node/internal"
)
type (
// BasicChecker is an interface of the basic ACL control tool.
BasicChecker interface {
// Action returns true if request is allowed for this target.
Action(uint32, object.RequestType, acl.Target) (bool, error)
// Bearer returns true if bearer token is allowed for this request.
Bearer(uint32, object.RequestType) (bool, error)
// Extended returns true if extended ACL is allowed for this.
Extended(uint32) bool
// Sticky returns true if sticky bit is set.
Sticky(uint32) bool
}
// BasicACLChecker performs basic ACL check.
BasicACLChecker struct{}
// MaskedBasicACLChecker performs all basic ACL checks, but applying
// mask on ACL first. It is useful, when some bits must be always
// set or unset.
MaskedBasicACLChecker struct {
BasicACLChecker
andMask uint32
orMask uint32
}
nibble struct {
value uint32
}
)
const (
errUnknownRequest = internal.Error("unknown request type")
errUnknownTarget = internal.Error("unknown target type")
)
const (
aclFinalBit = 0x10000000 // 29th bit
aclStickyBit = 0x20000000 // 30th bit
nibbleBBit = 0x1
nibbleOBit = 0x2
nibbleSBit = 0x4
nibbleUBit = 0x8
// DefaultAndFilter is a default AND mask of basic ACL value of container.
DefaultAndFilter = 0xFFFFFFFF
)
var (
nibbleOffset = map[object.RequestType]uint32{
object.RequestGet: 0,
object.RequestHead: 1 * 4,
object.RequestPut: 2 * 4,
object.RequestDelete: 3 * 4,
object.RequestSearch: 4 * 4,
object.RequestRange: 5 * 4,
object.RequestRangeHash: 6 * 4,
}
)
// Action returns true if request is allowed for target.
func (c *BasicACLChecker) Action(rule uint32, req object.RequestType, t acl.Target) (bool, error) {
n, err := fetchNibble(rule, req)
if err != nil {
return false, err
}
switch t {
case acl.Target_User:
return n.U(), nil
case acl.Target_System:
return n.S(), nil
case acl.Target_Others:
return n.O(), nil
default:
return false, errUnknownTarget
}
}
// Bearer returns true if bearer token is allowed to use for this request
// as source of extended ACL.
func (c *BasicACLChecker) Bearer(rule uint32, req object.RequestType) (bool, error) {
n, err := fetchNibble(rule, req)
if err != nil {
return false, err
}
return n.B(), nil
}
// Extended returns true if extended ACL stored in the container are allowed
// to use.
func (c *BasicACLChecker) Extended(rule uint32) bool {
return rule&aclFinalBit != aclFinalBit
}
// Sticky returns true if container is not allowed to store objects with
// owners different from request owner.
func (c *BasicACLChecker) Sticky(rule uint32) bool {
return rule&aclStickyBit == aclStickyBit
}
func fetchNibble(rule uint32, req object.RequestType) (*nibble, error) {
offset, ok := nibbleOffset[req]
if !ok {
return nil, errUnknownRequest
}
return &nibble{value: (rule >> offset) & 0xf}, nil
}
// B returns true if `Bearer` bit set in the nibble.
func (n *nibble) B() bool { return n.value&nibbleBBit == nibbleBBit }
// O returns true if `Others` bit set in the nibble.
func (n *nibble) O() bool { return n.value&nibbleOBit == nibbleOBit }
// S returns true if `System` bit set in the nibble.
func (n *nibble) S() bool { return n.value&nibbleSBit == nibbleSBit }
// U returns true if `User` bit set in the nibble.
func (n *nibble) U() bool { return n.value&nibbleUBit == nibbleUBit }
// NewMaskedBasicACLChecker returns BasicChecker that applies predefined
// bit mask on basic ACL value.
func NewMaskedBasicACLChecker(or, and uint32) BasicChecker {
return MaskedBasicACLChecker{
BasicACLChecker: BasicACLChecker{},
andMask: and,
orMask: or,
}
}
// Action returns true if request is allowed for target.
func (c MaskedBasicACLChecker) Action(rule uint32, req object.RequestType, t acl.Target) (bool, error) {
rule |= c.orMask
rule &= c.andMask
return c.BasicACLChecker.Action(rule, req, t)
}
// Bearer returns true if bearer token is allowed to use for this request
// as source of extended ACL.
func (c MaskedBasicACLChecker) Bearer(rule uint32, req object.RequestType) (bool, error) {
rule |= c.orMask
rule &= c.andMask
return c.BasicACLChecker.Bearer(rule, req)
}
// Extended returns true if extended ACL stored in the container are allowed
// to use.
func (c MaskedBasicACLChecker) Extended(rule uint32) bool {
rule |= c.orMask
rule &= c.andMask
return c.BasicACLChecker.Extended(rule)
}
// Sticky returns true if container is not allowed to store objects with
// owners different from request owner.
func (c MaskedBasicACLChecker) Sticky(rule uint32) bool {
rule |= c.orMask
rule &= c.andMask
return c.BasicACLChecker.Sticky(rule)
}

116
lib/acl/basic_test.go Normal file
View file

@ -0,0 +1,116 @@
package acl
import (
"math/bits"
"testing"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/stretchr/testify/require"
)
func TestBasicACLChecker(t *testing.T) {
reqs := []object.RequestType{
object.RequestGet,
object.RequestHead,
object.RequestPut,
object.RequestDelete,
object.RequestSearch,
object.RequestRange,
object.RequestRangeHash,
}
targets := []acl.Target{
acl.Target_Others,
acl.Target_System,
acl.Target_User,
}
checker := new(BasicACLChecker)
t.Run("verb permissions", func(t *testing.T) {
mask := uint32(1)
for i := range reqs {
res, err := checker.Bearer(mask, reqs[i])
require.NoError(t, err)
require.True(t, res)
mask = bits.Reverse32(mask)
res, err = checker.Bearer(mask, reqs[i])
require.NoError(t, err)
require.False(t, res)
mask = bits.Reverse32(mask)
for j := range targets {
mask <<= 1
res, err = checker.Action(mask, reqs[i], targets[j])
require.NoError(t, err)
require.True(t, res)
mask = bits.Reverse32(mask)
res, err = checker.Action(mask, reqs[i], targets[j])
require.NoError(t, err)
require.False(t, res)
mask = bits.Reverse32(mask)
}
mask <<= 1
}
})
t.Run("unknown verb", func(t *testing.T) {
mask := uint32(1)
_, err := checker.Bearer(mask, -1)
require.Error(t, err)
mask = 2
_, err = checker.Action(mask, -1, acl.Target_Others)
require.Error(t, err)
})
t.Run("unknown action", func(t *testing.T) {
mask := uint32(2)
_, err := checker.Action(mask, object.RequestGet, -1)
require.Error(t, err)
})
t.Run("extended acl permission", func(t *testing.T) {
// set F-bit
mask := uint32(0) | aclFinalBit
require.False(t, checker.Extended(mask))
// unset F-bit
mask = bits.Reverse32(mask)
require.True(t, checker.Extended(mask))
})
t.Run("sticky bit permission", func(t *testing.T) {
mask := uint32(0x20000000)
require.True(t, checker.Sticky(mask))
mask = bits.Reverse32(mask)
require.False(t, checker.Sticky(mask))
})
}
// todo: add tests like in basic acl checker
func TestNeoFSMaskedBasicACLChecker(t *testing.T) {
const orFilter = 0x04040444 // this OR filter will be used in neofs-node
checker := NewMaskedBasicACLChecker(orFilter, DefaultAndFilter)
reqs := []object.RequestType{
object.RequestGet,
object.RequestHead,
object.RequestPut,
object.RequestSearch,
object.RequestRangeHash,
}
for i := range reqs {
res, err := checker.Action(0, reqs[i], acl.Target_System)
require.NoError(t, err)
require.True(t, res)
}
}

129
lib/acl/binary.go Normal file
View file

@ -0,0 +1,129 @@
package acl
import (
"context"
"encoding/binary"
"io"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-node/internal"
)
// BinaryEACLKey is a binary EACL storage key.
type BinaryEACLKey struct {
cid refs.CID
}
// BinaryEACLValue is a binary EACL storage value.
type BinaryEACLValue struct {
eacl []byte
sig []byte
}
// BinaryExtendedACLSource is an interface of storage of binary extended ACL tables with read access.
type BinaryExtendedACLSource interface {
// Must return binary extended ACL table by key.
GetBinaryEACL(context.Context, BinaryEACLKey) (BinaryEACLValue, error)
}
// BinaryExtendedACLStore is an interface of storage of binary extended ACL tables.
type BinaryExtendedACLStore interface {
BinaryExtendedACLSource
// Must store binary extended ACL table for key.
PutBinaryEACL(context.Context, BinaryEACLKey, BinaryEACLValue) error
}
// ErrNilBinaryExtendedACLStore is returned by function that expect a non-nil
// BinaryExtendedACLStore, but received nil.
const ErrNilBinaryExtendedACLStore = internal.Error("binary extended ACL store is nil")
const sliceLenSize = 4
var eaclEndianness = binary.BigEndian
// CID is a container ID getter.
func (s BinaryEACLKey) CID() refs.CID {
return s.cid
}
// SetCID is a container ID setter.
func (s *BinaryEACLKey) SetCID(v refs.CID) {
s.cid = v
}
// EACL is a binary extended ACL table getter.
func (s BinaryEACLValue) EACL() []byte {
return s.eacl
}
// SetEACL is a binary extended ACL table setter.
func (s *BinaryEACLValue) SetEACL(v []byte) {
s.eacl = v
}
// Signature is an EACL signature getter.
func (s BinaryEACLValue) Signature() []byte {
return s.sig
}
// SetSignature is an EACL signature setter.
func (s *BinaryEACLValue) SetSignature(v []byte) {
s.sig = v
}
// MarshalBinary returns a binary representation of BinaryEACLValue.
func (s BinaryEACLValue) MarshalBinary() ([]byte, error) {
data := make([]byte, sliceLenSize+len(s.eacl)+sliceLenSize+len(s.sig))
off := 0
eaclEndianness.PutUint32(data[off:], uint32(len(s.eacl)))
off += sliceLenSize
off += copy(data[off:], s.eacl)
eaclEndianness.PutUint32(data[off:], uint32(len(s.sig)))
off += sliceLenSize
copy(data[off:], s.sig)
return data, nil
}
// UnmarshalBinary unmarshals BinaryEACLValue from bytes.
func (s *BinaryEACLValue) UnmarshalBinary(data []byte) (err error) {
err = io.ErrUnexpectedEOF
off := 0
if len(data[off:]) < sliceLenSize {
return
}
aclLn := eaclEndianness.Uint32(data[off:])
off += 4
if uint32(len(data[off:])) < aclLn {
return
}
s.eacl = make([]byte, aclLn)
off += copy(s.eacl, data[off:])
if len(data[off:]) < sliceLenSize {
return
}
sigLn := eaclEndianness.Uint32(data[off:])
off += 4
if uint32(len(data[off:])) < sigLn {
return
}
s.sig = make([]byte, sigLn)
copy(s.sig, data[off:])
return nil
}

27
lib/acl/binary_test.go Normal file
View file

@ -0,0 +1,27 @@
package acl
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestBinaryEACLValue(t *testing.T) {
s := BinaryEACLValue{}
eacl := []byte{1, 2, 3}
s.SetEACL(eacl)
require.Equal(t, eacl, s.EACL())
sig := []byte{4, 5, 6}
s.SetSignature(sig)
require.Equal(t, sig, s.Signature())
data, err := s.MarshalBinary()
require.NoError(t, err)
s2 := BinaryEACLValue{}
require.NoError(t, s2.UnmarshalBinary(data))
require.Equal(t, s, s2)
}

29
lib/acl/extended.go Normal file
View file

@ -0,0 +1,29 @@
package acl
import (
"context"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-api-go/refs"
)
// TypedHeaderSource is a various types of header set interface.
type TypedHeaderSource interface {
// Must return list of Header of particular type.
// Must return false if there is no ability to compose header list.
HeadersOfType(acl.HeaderType) ([]acl.Header, bool)
}
// ExtendedACLSource is an interface of storage of extended ACL tables with read access.
type ExtendedACLSource interface {
// Must return extended ACL table by container ID key.
GetExtendedACLTable(context.Context, refs.CID) (acl.ExtendedACLTable, error)
}
// ExtendedACLStore is an interface of storage of extended ACL tables.
type ExtendedACLStore interface {
ExtendedACLSource
// Must store extended ACL table for container ID key.
PutExtendedACLTable(context.Context, refs.CID, acl.ExtendedACLTable) error
}

234
lib/acl/header.go Normal file
View file

@ -0,0 +1,234 @@
package acl
import (
"strconv"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/service"
)
type objectHeaderSource struct {
obj *object.Object
}
type typedHeader struct {
n string
v string
t acl.HeaderType
}
type extendedHeadersWrapper struct {
hdrSrc service.ExtendedHeadersSource
}
type typedExtendedHeader struct {
hdr service.ExtendedHeader
}
func newTypedObjSysHdr(name, value string) acl.TypedHeader {
return &typedHeader{
n: name,
v: value,
t: acl.HdrTypeObjSys,
}
}
// Name is a name field getter.
func (s typedHeader) Name() string {
return s.n
}
// Value is a value field getter.
func (s typedHeader) Value() string {
return s.v
}
// HeaderType is a type field getter.
func (s typedHeader) HeaderType() acl.HeaderType {
return s.t
}
// TypedHeaderSourceFromObject wraps passed object and returns TypedHeaderSource interface.
func TypedHeaderSourceFromObject(obj *object.Object) TypedHeaderSource {
return &objectHeaderSource{
obj: obj,
}
}
// HeaderOfType gathers object headers of passed type and returns Header list.
//
// If value of some header can not be calculated (e.g. nil extended header), it does not appear in list.
//
// Always returns true.
func (s objectHeaderSource) HeadersOfType(typ acl.HeaderType) ([]acl.Header, bool) {
if s.obj == nil {
return nil, true
}
var res []acl.Header
switch typ {
case acl.HdrTypeObjUsr:
objHeaders := s.obj.GetHeaders()
res = make([]acl.Header, 0, len(objHeaders)) // 7 system header fields
for i := range objHeaders {
if h := newTypedObjectExtendedHeader(objHeaders[i]); h != nil {
res = append(res, h)
}
}
case acl.HdrTypeObjSys:
res = make([]acl.Header, 0, 7)
sysHdr := s.obj.GetSystemHeader()
created := sysHdr.GetCreatedAt()
res = append(res,
// ID
newTypedObjSysHdr(
acl.HdrObjSysNameID,
sysHdr.ID.String(),
),
// CID
newTypedObjSysHdr(
acl.HdrObjSysNameCID,
sysHdr.CID.String(),
),
// OwnerID
newTypedObjSysHdr(
acl.HdrObjSysNameOwnerID,
sysHdr.OwnerID.String(),
),
// Version
newTypedObjSysHdr(
acl.HdrObjSysNameVersion,
strconv.FormatUint(sysHdr.GetVersion(), 10),
),
// PayloadLength
newTypedObjSysHdr(
acl.HdrObjSysNamePayloadLength,
strconv.FormatUint(sysHdr.GetPayloadLength(), 10),
),
// CreatedAt.UnitTime
newTypedObjSysHdr(
acl.HdrObjSysNameCreatedUnix,
strconv.FormatUint(uint64(created.GetUnixTime()), 10),
),
// CreatedAt.Epoch
newTypedObjSysHdr(
acl.HdrObjSysNameCreatedEpoch,
strconv.FormatUint(created.GetEpoch(), 10),
),
)
}
return res, true
}
func newTypedObjectExtendedHeader(h object.Header) acl.TypedHeader {
val := h.GetValue()
if val == nil {
return nil
}
res := new(typedHeader)
res.t = acl.HdrTypeObjSys
switch hdr := val.(type) {
case *object.Header_UserHeader:
if hdr.UserHeader == nil {
return nil
}
res.t = acl.HdrTypeObjUsr
res.n = hdr.UserHeader.GetKey()
res.v = hdr.UserHeader.GetValue()
case *object.Header_Link:
if hdr.Link == nil {
return nil
}
switch hdr.Link.GetType() {
case object.Link_Previous:
res.n = acl.HdrObjSysLinkPrev
case object.Link_Next:
res.n = acl.HdrObjSysLinkNext
case object.Link_Child:
res.n = acl.HdrObjSysLinkChild
case object.Link_Parent:
res.n = acl.HdrObjSysLinkPar
case object.Link_StorageGroup:
res.n = acl.HdrObjSysLinkSG
default:
return nil
}
res.v = hdr.Link.ID.String()
default:
return nil
}
return res
}
// TypedHeaderSourceFromExtendedHeaders wraps passed ExtendedHeadersSource and returns TypedHeaderSource interface.
func TypedHeaderSourceFromExtendedHeaders(hdrSrc service.ExtendedHeadersSource) TypedHeaderSource {
return &extendedHeadersWrapper{
hdrSrc: hdrSrc,
}
}
// Name returns the result of Key method.
func (s typedExtendedHeader) Name() string {
return s.hdr.Key()
}
// Value returns the result of Value method.
func (s typedExtendedHeader) Value() string {
return s.hdr.Value()
}
// HeaderType always returns HdrTypeRequest.
func (s typedExtendedHeader) HeaderType() acl.HeaderType {
return acl.HdrTypeRequest
}
// TypedHeaders gathers extended request headers and returns TypedHeader list.
//
// Nil headers are ignored.
//
// Always returns true.
func (s extendedHeadersWrapper) HeadersOfType(typ acl.HeaderType) ([]acl.Header, bool) {
if s.hdrSrc == nil {
return nil, true
}
var res []acl.Header
if typ == acl.HdrTypeRequest {
hs := s.hdrSrc.ExtendedHeaders()
res = make([]acl.Header, 0, len(hs))
for i := range hs {
if hs[i] == nil {
continue
}
res = append(res, &typedExtendedHeader{
hdr: hs[i],
})
}
}
return res, true
}

60
lib/acl/headers_test.go Normal file
View file

@ -0,0 +1,60 @@
package acl
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/stretchr/testify/require"
)
func TestNewTypedObjectExtendedHeader(t *testing.T) {
var res acl.TypedHeader
hdr := object.Header{}
// nil value
require.Nil(t, newTypedObjectExtendedHeader(hdr))
// UserHeader
{
key := "key"
val := "val"
hdr.Value = &object.Header_UserHeader{
UserHeader: &object.UserHeader{
Key: key,
Value: val,
},
}
res = newTypedObjectExtendedHeader(hdr)
require.Equal(t, acl.HdrTypeObjUsr, res.HeaderType())
require.Equal(t, key, res.Name())
require.Equal(t, val, res.Value())
}
{ // Link
link := new(object.Link)
link.ID = object.ID{1, 2, 3}
hdr.Value = &object.Header_Link{
Link: link,
}
check := func(lt object.Link_Type, name string) {
link.Type = lt
res = newTypedObjectExtendedHeader(hdr)
require.Equal(t, acl.HdrTypeObjSys, res.HeaderType())
require.Equal(t, name, res.Name())
require.Equal(t, link.ID.String(), res.Value())
}
check(object.Link_Previous, acl.HdrObjSysLinkPrev)
check(object.Link_Next, acl.HdrObjSysLinkNext)
check(object.Link_Parent, acl.HdrObjSysLinkPar)
check(object.Link_Child, acl.HdrObjSysLinkChild)
check(object.Link_StorageGroup, acl.HdrObjSysLinkSG)
}
}

94
lib/acl/match.go Normal file
View file

@ -0,0 +1,94 @@
package acl
import (
"github.com/nspcc-dev/neofs-api-go/acl"
)
// Maps MatchType to corresponding function.
// 1st argument of function - header value, 2nd - header filter.
var mMatchFns = map[acl.MatchType]func(acl.Header, acl.Header) bool{
acl.StringEqual: stringEqual,
acl.StringNotEqual: stringNotEqual,
}
const (
mResUndefined = iota
mResMatch
mResMismatch
)
// MatchFilters checks if passed source carry at least one header that satisfies passed filters.
//
// Nil header does not satisfy any filter. Any header does not satisfy nil filter.
//
// Returns mResMismatch if passed TypedHeaderSource is nil.
// Returns mResMatch if passed filters are empty.
//
// If headers for some of the HeaderType could not be composed, mResUndefined returns.
func MatchFilters(src TypedHeaderSource, filters []acl.HeaderFilter) int {
if src == nil {
return mResMismatch
} else if len(filters) == 0 {
return mResMatch
}
matched := 0
for _, filter := range filters {
// prevent NPE
if filter == nil {
continue
}
headers, ok := src.HeadersOfType(filter.HeaderType())
if !ok {
return mResUndefined
}
// get headers of filtering type
for _, header := range headers {
// prevent NPE
if header == nil {
continue
}
// check header name
if header.Name() != filter.Name() {
continue
}
// get match function
matchFn, ok := mMatchFns[filter.MatchType()]
if !ok {
continue
}
// check match
if !matchFn(header, filter) {
continue
}
// increment match counter
matched++
break
}
}
res := mResMismatch
if matched >= len(filters) {
res = mResMatch
}
return res
}
func stringEqual(header, filter acl.Header) bool {
return header.Value() == filter.Value()
}
func stringNotEqual(header, filter acl.Header) bool {
return header.Value() != filter.Value()
}

192
lib/acl/match_test.go Normal file
View file

@ -0,0 +1,192 @@
package acl
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/stretchr/testify/require"
)
type testTypedHeader struct {
t acl.HeaderType
k string
v string
}
type testHeaderSrc struct {
hs []acl.TypedHeader
}
type testHeaderFilter struct {
acl.TypedHeader
t acl.MatchType
}
func (s testHeaderFilter) MatchType() acl.MatchType {
return s.t
}
func (s testHeaderSrc) HeadersOfType(typ acl.HeaderType) ([]acl.Header, bool) {
res := make([]acl.Header, 0, len(s.hs))
for i := range s.hs {
if s.hs[i].HeaderType() == typ {
res = append(res, s.hs[i])
}
}
return res, true
}
func (s testTypedHeader) Name() string {
return s.k
}
func (s testTypedHeader) Value() string {
return s.v
}
func (s testTypedHeader) HeaderType() acl.HeaderType {
return s.t
}
func TestMatchFilters(t *testing.T) {
// nil TypedHeaderSource
require.Equal(t, mResMismatch, MatchFilters(nil, nil))
// empty HeaderFilter list
require.Equal(t, mResMatch, MatchFilters(new(testHeaderSrc), nil))
k := "key"
v := "value"
ht := acl.HeaderType(1)
items := []struct {
// list of Key-Value-HeaderType for headers construction
hs []interface{}
// list of Key-Value-HeaderType-MatchType for filters construction
fs []interface{}
exp int
}{
{ // different HeaderType
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v, ht + 1, acl.StringEqual,
},
exp: mResMismatch,
},
{ // different keys
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k + "1", v, ht, acl.StringEqual,
},
exp: mResMismatch,
},
{ // equal values, StringEqual
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v, ht, acl.StringEqual,
},
exp: mResMatch,
},
{ // equal values, StringNotEqual
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v, ht, acl.StringNotEqual,
},
exp: mResMismatch,
},
{ // not equal values, StringEqual
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v + "1", ht, acl.StringEqual,
},
exp: mResMismatch,
},
{ // not equal values, StringNotEqual
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v + "1", ht, acl.StringNotEqual,
},
exp: mResMatch,
},
{ // one header, two filters
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v + "1", ht, acl.StringNotEqual,
k, v, ht, acl.StringEqual,
},
exp: mResMatch,
},
{ // two headers, one filter
hs: []interface{}{
k, v + "1", ht,
k, v, ht,
},
fs: []interface{}{
k, v, ht, acl.StringEqual,
},
exp: mResMatch,
},
{
hs: []interface{}{
k, v + "1", acl.HdrTypeRequest,
k, v, acl.HdrTypeObjUsr,
},
fs: []interface{}{
k, v, acl.HdrTypeRequest, acl.StringNotEqual,
k, v, acl.HdrTypeObjUsr, acl.StringEqual,
},
exp: mResMatch,
},
}
for _, item := range items {
headers := make([]acl.TypedHeader, 0)
for i := 0; i < len(item.hs); i += 3 {
headers = append(headers, &testTypedHeader{
t: item.hs[i+2].(acl.HeaderType),
k: item.hs[i].(string),
v: item.hs[i+1].(string),
})
}
filters := make([]acl.HeaderFilter, 0)
for i := 0; i < len(item.fs); i += 4 {
filters = append(filters, &testHeaderFilter{
TypedHeader: &testTypedHeader{
t: item.fs[i+2].(acl.HeaderType),
k: item.fs[i].(string),
v: item.fs[i+1].(string),
},
t: item.fs[i+3].(acl.MatchType),
})
}
require.Equal(t,
item.exp,
MatchFilters(
&testHeaderSrc{
hs: headers,
},
filters,
),
)
}
}

View file

@ -0,0 +1,31 @@
package event
// Type is a notification event enumeration type.
type Type string
// Event is an interface that is
// provided by Neo:Morph event structures.
type Event interface {
MorphEvent()
}
// Equal compares two Type values and
// returns true if they are equal.
func (t Type) Equal(t2 Type) bool {
return string(t) == string(t2)
}
// String returns casted to string Type.
func (t Type) String() string {
return string(t)
}
// TypeFromBytes converts bytes slice to Type.
func TypeFromBytes(data []byte) Type {
return Type(data)
}
// TypeFromString converts string to Type.
func TypeFromString(str string) Type {
return Type(str)
}

View file

@ -0,0 +1,22 @@
package event
// Handler is an Event processing function.
type Handler func(Event)
// HandlerInfo is a structure that groups
// the parameters of the handler of particular
// contract event.
type HandlerInfo struct {
scriptHashWithType
h Handler
}
// SetHandler is an event handler setter.
func (s *HandlerInfo) SetHandler(v Handler) {
s.h = v
}
func (s HandlerInfo) handler() Handler {
return s.h
}

View file

@ -0,0 +1,309 @@
package event
import (
"context"
"sync"
"github.com/nspcc-dev/neo-go/pkg/rpc/response/result"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/blockchain/goclient"
"github.com/nspcc-dev/neofs-node/lib/blockchain/subscriber"
"github.com/pkg/errors"
"go.uber.org/zap"
)
// Listener is an interface of smart contract notification event listener.
type Listener interface {
// Must start the event listener.
//
// Must listen to events with the parser installed.
//
// Must return an error if event listening could not be started.
Listen(context.Context)
// Must set the parser of particular contract event.
//
// Parser of each event must be set once. All parsers must be set before Listen call.
//
// Must ignore nil parsers and all calls after listener has been started.
SetParser(ParserInfo)
// Must register the event handler for particular notification event of contract.
//
// The specified handler must be called after each capture and parsing of the event
//
// Must ignore nil handlers.
RegisterHandler(HandlerInfo)
}
// ListenerParams is a group of parameters
// for Listener constructor.
type ListenerParams struct {
Logger *zap.Logger
Subscriber subscriber.Subscriber
}
type listener struct {
mtx *sync.RWMutex
once *sync.Once
started bool
parsers map[scriptHashWithType]Parser
handlers map[scriptHashWithType][]Handler
log *zap.Logger
subscriber subscriber.Subscriber
}
const (
newListenerFailMsg = "could not instantiate Listener"
errNilLogger = internal.Error("nil logger")
errNilSubscriber = internal.Error("nil event subscriber")
)
// Listen starts the listening for events with registered handlers.
//
// Executes once, all subsequent calls do nothing.
//
// Returns an error if listener was already started.
func (s listener) Listen(ctx context.Context) {
s.once.Do(func() {
if err := s.listen(ctx); err != nil {
s.log.Error("could not start listen to events",
zap.String("error", err.Error()),
)
}
})
}
func (s listener) listen(ctx context.Context) error {
// create the list of listening contract hashes
hashes := make([]util.Uint160, 0)
// fill the list with the contracts with set event parsers.
s.mtx.RLock()
for hashType := range s.parsers {
scHash := hashType.scriptHash()
// prevent repetitions
for _, hash := range hashes {
if hash.Equals(scHash) {
continue
}
}
hashes = append(hashes, hashType.scriptHash())
}
// mark listener as started
s.started = true
s.mtx.RUnlock()
chEvent, err := s.subscriber.SubscribeForNotification(hashes...)
if err != nil {
return err
}
s.listenLoop(ctx, chEvent)
return nil
}
func (s listener) listenLoop(ctx context.Context, chEvent <-chan *result.NotificationEvent) {
loop:
for {
select {
case <-ctx.Done():
s.log.Warn("stop event listener by context",
zap.String("error", ctx.Err().Error()),
)
break loop
case notifyEvent, ok := <-chEvent:
if !ok {
s.log.Warn("stop event listener by channel")
break loop
} else if notifyEvent == nil {
s.log.Warn("nil notification event was caught")
continue loop
}
s.parseAndHandle(notifyEvent)
}
}
}
func (s listener) parseAndHandle(notifyEvent *result.NotificationEvent) {
log := s.log.With(
zap.String("script hash LE", notifyEvent.Contract.StringLE()),
)
// stack item must be an array of items
arr, err := goclient.ArrayFromStackParameter(notifyEvent.Item)
if err != nil {
log.Warn("stack item is not an array type",
zap.String("error", err.Error()),
)
return
} else if len(arr) == 0 {
log.Warn("stack item array is empty")
return
}
// first item must be a byte array
typBytes, err := goclient.BytesFromStackParameter(arr[0])
if err != nil {
log.Warn("first array item is not a byte array",
zap.String("error", err.Error()),
)
return
}
// calculate event type from bytes
typEvent := TypeFromBytes(typBytes)
log = log.With(
zap.Stringer("event type", typEvent),
)
// get the event parser
keyEvent := scriptHashWithType{}
keyEvent.SetScriptHash(notifyEvent.Contract)
keyEvent.SetType(typEvent)
s.mtx.RLock()
parser, ok := s.parsers[keyEvent]
s.mtx.RUnlock()
if !ok {
log.Warn("event parser not set")
return
}
// parse the notification event
event, err := parser(arr[1:])
if err != nil {
log.Warn("could not parse notification event",
zap.String("error", err.Error()),
)
return
}
// handler the event
s.mtx.RLock()
handlers := s.handlers[keyEvent]
s.mtx.RUnlock()
if len(handlers) == 0 {
log.Info("handlers for parsed notification event were not registered",
zap.Any("event", event),
)
return
}
for _, handler := range handlers {
handler(event)
}
}
// SetParser sets the parser of particular contract event.
//
// Ignores nil and already set parsers.
// Ignores the parser if listener is started.
func (s listener) SetParser(p ParserInfo) {
log := s.log.With(
zap.String("script hash LE", p.scriptHash().StringLE()),
zap.Stringer("event type", p.getType()),
)
parser := p.parser()
if parser == nil {
log.Info("ignore nil event parser")
return
}
s.mtx.Lock()
defer s.mtx.Unlock()
// check if the listener was started
if s.started {
log.Warn("listener has been already started, ignore parser")
return
}
// add event parser
if _, ok := s.parsers[p.scriptHashWithType]; !ok {
s.parsers[p.scriptHashWithType] = p.parser()
}
log.Info("registered new event parser")
}
// RegisterHandler registers the handler for particular notification event of contract.
//
// Ignores nil handlers.
// Ignores handlers of event without parser.
func (s listener) RegisterHandler(p HandlerInfo) {
log := s.log.With(
zap.String("script hash LE", p.scriptHash().StringLE()),
zap.Stringer("event type", p.getType()),
)
handler := p.handler()
if handler == nil {
log.Warn("ignore nil event handler")
return
}
// check if parser was set
s.mtx.RLock()
_, ok := s.parsers[p.scriptHashWithType]
s.mtx.RUnlock()
if !ok {
log.Warn("ignore handler of event w/o parser")
return
}
// add event handler
s.mtx.Lock()
s.handlers[p.scriptHashWithType] = append(
s.handlers[p.scriptHashWithType],
p.handler(),
)
s.mtx.Unlock()
log.Info("registered new event handler")
}
// NewListener create the notification event listener instance and returns Listener interface.
func NewListener(p ListenerParams) (Listener, error) {
switch {
case p.Logger == nil:
return nil, errors.Wrap(errNilLogger, newListenerFailMsg)
case p.Subscriber == nil:
return nil, errors.Wrap(errNilSubscriber, newListenerFailMsg)
}
return &listener{
mtx: new(sync.RWMutex),
once: new(sync.Once),
parsers: make(map[scriptHashWithType]Parser),
handlers: make(map[scriptHashWithType][]Handler),
log: p.Logger,
subscriber: p.Subscriber,
}, nil
}

View file

@ -0,0 +1,39 @@
package netmap
import (
"github.com/nspcc-dev/neo-go/pkg/smartcontract"
"github.com/nspcc-dev/neofs-node/lib/blockchain/event"
"github.com/nspcc-dev/neofs-node/lib/blockchain/goclient"
"github.com/pkg/errors"
)
// NewEpoch is a new epoch Neo:Morph event.
type NewEpoch struct {
num uint64
}
// MorphEvent implements Neo:Morph Event interface.
func (NewEpoch) MorphEvent() {}
// EpochNumber returns new epoch number.
func (s NewEpoch) EpochNumber() uint64 {
return s.num
}
// ParseNewEpoch is a parser of new epoch notification event.
//
// Result is type of NewEpoch.
func ParseNewEpoch(prms []smartcontract.Parameter) (event.Event, error) {
if ln := len(prms); ln != 1 {
return nil, event.WrongNumberOfParameters(1, ln)
}
prmEpochNum, err := goclient.IntFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get integer epoch number")
}
return NewEpoch{
num: uint64(prmEpochNum),
}, nil
}

View file

@ -0,0 +1,47 @@
package netmap
import (
"testing"
"github.com/nspcc-dev/neo-go/pkg/smartcontract"
"github.com/nspcc-dev/neofs-node/lib/blockchain/event"
"github.com/stretchr/testify/require"
)
func TestParseNewEpoch(t *testing.T) {
t.Run("wrong number of parameters", func(t *testing.T) {
prms := []smartcontract.Parameter{
{},
{},
}
_, err := ParseNewEpoch(prms)
require.EqualError(t, err, event.WrongNumberOfParameters(1, len(prms)).Error())
})
t.Run("wrong first parameter type", func(t *testing.T) {
_, err := ParseNewEpoch([]smartcontract.Parameter{
{
Type: smartcontract.ByteArrayType,
},
})
require.Error(t, err)
})
t.Run("correct behavior", func(t *testing.T) {
epochNum := uint64(100)
ev, err := ParseNewEpoch([]smartcontract.Parameter{
{
Type: smartcontract.IntegerType,
Value: int64(epochNum),
},
})
require.NoError(t, err)
require.Equal(t, NewEpoch{
num: epochNum,
}, ev)
})
}

View file

@ -0,0 +1,53 @@
package event
import (
"github.com/nspcc-dev/neo-go/pkg/smartcontract"
"github.com/pkg/errors"
)
// Parser is a function that constructs Event
// from the StackItem list.
type Parser func([]smartcontract.Parameter) (Event, error)
// ParserInfo is a structure that groups
// the parameters of particular contract
// notification event parser.
type ParserInfo struct {
scriptHashWithType
p Parser
}
type wrongPrmNumber struct {
exp, act int
}
// WrongNumberOfParameters returns an error about wrong number of smart contract parameters.
func WrongNumberOfParameters(exp, act int) error {
return &wrongPrmNumber{
exp: exp,
act: act,
}
}
func (s wrongPrmNumber) Error() string {
return errors.Errorf("wrong parameter count: expected %d, has %d", s.exp, s.act).Error()
}
// SetParser is an event parser setter.
func (s *ParserInfo) SetParser(v Parser) {
s.p = v
}
func (s ParserInfo) parser() Parser {
return s.p
}
// SetType is an event type setter.
func (s *ParserInfo) SetType(v Type) {
s.typ = v
}
func (s ParserInfo) getType() Type {
return s.typ
}

View file

@ -0,0 +1,34 @@
package event
import "github.com/nspcc-dev/neo-go/pkg/util"
type scriptHashValue struct {
hash util.Uint160
}
type typeValue struct {
typ Type
}
type scriptHashWithType struct {
scriptHashValue
typeValue
}
// SetScriptHash is a script hash setter.
func (s *scriptHashValue) SetScriptHash(v util.Uint160) {
s.hash = v
}
func (s scriptHashValue) scriptHash() util.Uint160 {
return s.hash
}
// SetType is an event type setter.
func (s *typeValue) SetType(v Type) {
s.typ = v
}
func (s typeValue) getType() Type {
return s.typ
}

View file

@ -0,0 +1,190 @@
package goclient
import (
"context"
"crypto/ecdsa"
"encoding/hex"
"time"
"github.com/nspcc-dev/neo-go/pkg/config/netmode"
"github.com/nspcc-dev/neo-go/pkg/core/transaction"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/rpc/client"
sc "github.com/nspcc-dev/neo-go/pkg/smartcontract"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/wallet"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/pkg/errors"
"go.uber.org/zap"
)
type (
// Params is a group of Client's constructor parameters.
Params struct {
Log *zap.Logger
Key *ecdsa.PrivateKey
Endpoint string
Magic netmode.Magic
DialTimeout time.Duration
}
// Client is a neo-go wrapper that provides smart-contract invocation interface.
Client struct {
log *zap.Logger
cli *client.Client
acc *wallet.Account
}
)
// ErrNilClient is returned by functions that expect
// a non-nil Client, but received nil.
const ErrNilClient = internal.Error("go client is nil")
// HaltState returned if TestInvoke function processed without panic.
const HaltState = "HALT"
// ErrMissingFee is returned by functions that expect
// a positive invocation fee, but received non-positive.
const ErrMissingFee = internal.Error("invocation fee must be positive")
var (
errNilParams = errors.New("chain/client: config was not provided to the constructor")
errNilLogger = errors.New("chain/client: logger was not provided to the constructor")
errNilKey = errors.New("chain/client: private key was not provided to the constructor")
)
// Invoke invokes contract method by sending transaction into blockchain.
// Supported args types: int64, string, util.Uint160, []byte and bool.
//
// If passed fee is non-positive, ErrMissingFee returns.
func (c *Client) Invoke(contract util.Uint160, fee util.Fixed8, method string, args ...interface{}) error {
var params []sc.Parameter
for i := range args {
param, err := toStackParameter(args[i])
if err != nil {
return err
}
params = append(params, param)
}
cosigner := []transaction.Cosigner{
{
Account: c.acc.PrivateKey().PublicKey().GetScriptHash(),
Scopes: transaction.Global,
},
}
resp, err := c.cli.InvokeFunction(contract, method, params, cosigner)
if err != nil {
return err
}
if len(resp.Script) == 0 {
return errors.New("chain/client: got empty invocation script from neo node")
}
script, err := hex.DecodeString(resp.Script)
if err != nil {
return errors.New("chain/client: can't decode invocation script from neo node")
}
txHash, err := c.cli.SignAndPushInvocationTx(script, c.acc, 0, fee, cosigner)
if err != nil {
return err
}
c.log.Debug("neo client invoke",
zap.String("method", method),
zap.Stringer("tx_hash", txHash))
return nil
}
// TestInvoke invokes contract method locally in neo-go node. This method should
// be used to read data from smart-contract.
func (c *Client) TestInvoke(contract util.Uint160, method string, args ...interface{}) ([]sc.Parameter, error) {
var params = make([]sc.Parameter, 0, len(args))
for i := range args {
p, err := toStackParameter(args[i])
if err != nil {
return nil, err
}
params = append(params, p)
}
cosigner := []transaction.Cosigner{
{
Account: c.acc.PrivateKey().PublicKey().GetScriptHash(),
Scopes: transaction.Global,
},
}
val, err := c.cli.InvokeFunction(contract, method, params, cosigner)
if err != nil {
return nil, err
}
if val.State != HaltState {
return nil, errors.Errorf("chain/client: contract execution finished with state %s", val.State)
}
return val.Stack, nil
}
// New is a Client constructor.
func New(ctx context.Context, p *Params) (*Client, error) {
switch {
case p == nil:
return nil, errNilParams
case p.Log == nil:
return nil, errNilLogger
case p.Key == nil:
return nil, errNilKey
}
privKeyBytes := crypto.MarshalPrivateKey(p.Key)
wif, err := keys.WIFEncode(privKeyBytes, keys.WIFVersion, true)
if err != nil {
return nil, err
}
account, err := wallet.NewAccountFromWIF(wif)
if err != nil {
return nil, err
}
cli, err := client.New(ctx, p.Endpoint, client.Options{
DialTimeout: p.DialTimeout,
Network: p.Magic,
})
if err != nil {
return nil, err
}
return &Client{log: p.Log, cli: cli, acc: account}, nil
}
func toStackParameter(value interface{}) (sc.Parameter, error) {
var result = sc.Parameter{
Value: value,
}
// todo: add more types
switch value.(type) {
case []byte:
result.Type = sc.ByteArrayType
case int64: // TODO: add other numerical types
result.Type = sc.IntegerType
default:
return result, errors.Errorf("chain/client: unsupported parameter %v", value)
}
return result, nil
}

View file

@ -0,0 +1,33 @@
package goclient
import (
"testing"
sc "github.com/nspcc-dev/neo-go/pkg/smartcontract"
"github.com/stretchr/testify/require"
)
func TestToStackParameter(t *testing.T) {
items := []struct {
value interface{}
expType sc.ParamType
}{
{
value: []byte{1, 2, 3},
expType: sc.ByteArrayType,
},
{
value: int64(100),
expType: sc.IntegerType,
},
}
for _, item := range items {
t.Run(item.expType.String()+" to stack parameter", func(t *testing.T) {
res, err := toStackParameter(item.value)
require.NoError(t, err)
require.Equal(t, item.expType, res.Type)
require.Equal(t, item.value, res.Value)
})
}
}

View file

@ -0,0 +1,131 @@
package goclient
import (
"encoding/binary"
sc "github.com/nspcc-dev/neo-go/pkg/smartcontract"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/pkg/errors"
)
/*
Use these function to parse stack parameters obtained from `TestInvoke`
function to native go types. You should know upfront return types of invoked
method.
*/
// BoolFromStackParameter receives boolean value from the value of a smart contract parameter.
func BoolFromStackParameter(param sc.Parameter) (bool, error) {
switch param.Type {
case sc.BoolType:
val, ok := param.Value.(bool)
if !ok {
return false, errors.Errorf("chain/client: can't convert %T to boolean", param.Value)
}
return val, nil
case sc.IntegerType:
val, ok := param.Value.(int64)
if !ok {
return false, errors.Errorf("chain/client: can't convert %T to boolean", param.Value)
}
return val > 0, nil
case sc.ByteArrayType:
val, ok := param.Value.([]byte)
if !ok {
return false, errors.Errorf("chain/client: can't convert %T to boolean", param.Value)
}
return len(val) != 0, nil
default:
return false, errors.Errorf("chain/client: %s is not a bool type", param.Type)
}
}
// IntFromStackParameter receives numerical value from the value of a smart contract parameter.
func IntFromStackParameter(param sc.Parameter) (int64, error) {
switch param.Type {
case sc.IntegerType:
val, ok := param.Value.(int64)
if !ok {
return 0, errors.Errorf("chain/client: can't convert %T to integer", param.Value)
}
return val, nil
case sc.ByteArrayType:
val, ok := param.Value.([]byte)
if !ok || len(val) > 8 {
return 0, errors.Errorf("chain/client: can't convert %T to integer", param.Value)
}
res := make([]byte, 8)
copy(res[:len(val)], val)
return int64(binary.LittleEndian.Uint64(res)), nil
default:
return 0, errors.Errorf("chain/client: %s is not an integer type", param.Type)
}
}
// BytesFromStackParameter receives binary value from the value of a smart contract parameter.
func BytesFromStackParameter(param sc.Parameter) ([]byte, error) {
if param.Type != sc.ByteArrayType {
return nil, errors.Errorf("chain/client: %s is not a byte array type", param.Type)
}
val, ok := param.Value.([]byte)
if !ok {
return nil, errors.Errorf("chain/client: can't convert %T to byte slice", param.Value)
}
return val, nil
}
// ArrayFromStackParameter returns the slice contract parameters from passed parameter.
//
// If passed parameter carries boolean false value, (nil, nil) returns.
func ArrayFromStackParameter(param sc.Parameter) ([]sc.Parameter, error) {
if param.Type == sc.BoolType && !param.Value.(bool) {
return nil, nil
}
if param.Type != sc.ArrayType {
return nil, errors.Errorf("chain/client: %s is not an array type", param.Type)
}
val, ok := param.Value.([]sc.Parameter)
if !ok {
return nil, errors.Errorf("chain/client: can't convert %T to parameter slice", param.Value)
}
return val, nil
}
// StringFromStackParameter receives string value from the value of a smart contract parameter.
func StringFromStackParameter(param sc.Parameter) (string, error) {
switch param.Type {
case sc.StringType:
val, ok := param.Value.(string)
if !ok {
return "", errors.Errorf("chain/client: can't convert %T to string", param.Value)
}
return val, nil
case sc.ByteArrayType:
val, ok := param.Value.([]byte)
if !ok {
return "", errors.Errorf("chain/client: can't convert %T to string", param.Value)
}
return string(val), nil
default:
return "", errors.Errorf("chain/client: %s is not a string type", param.Type)
}
}
// ReadStorage of the contract directly. Use it for debug, try to obtain
// smart-contract data from contract method with TestInvoke function.
func ReadStorage(c *Client, contract util.Uint160, key []byte) ([]byte, error) {
return c.cli.GetStorageByHash(contract, key)
}

View file

@ -0,0 +1,145 @@
package goclient
import (
"testing"
sc "github.com/nspcc-dev/neo-go/pkg/smartcontract"
"github.com/stretchr/testify/require"
)
var (
stringParam = sc.Parameter{
Type: sc.StringType,
Value: "Hello World",
}
intParam = sc.Parameter{
Type: sc.IntegerType,
Value: int64(1),
}
byteWithIntParam = sc.Parameter{
Type: sc.ByteArrayType,
Value: []byte{0x0a},
}
byteArrayParam = sc.Parameter{
Type: sc.ByteArrayType,
Value: []byte("Hello World"),
}
emptyByteArrayParam = sc.Parameter{
Type: sc.ByteArrayType,
Value: []byte{},
}
trueBoolParam = sc.Parameter{
Type: sc.BoolType,
Value: true,
}
falseBoolParam = sc.Parameter{
Type: sc.BoolType,
Value: false,
}
arrayParam = sc.Parameter{
Type: sc.ArrayType,
Value: []sc.Parameter{intParam, byteArrayParam},
}
)
func TestBoolFromStackParameter(t *testing.T) {
t.Run("true assert", func(t *testing.T) {
val, err := BoolFromStackParameter(trueBoolParam)
require.NoError(t, err)
require.True(t, val)
val, err = BoolFromStackParameter(intParam)
require.NoError(t, err)
require.True(t, val)
})
t.Run("false assert", func(t *testing.T) {
val, err := BoolFromStackParameter(falseBoolParam)
require.NoError(t, err)
require.False(t, val)
val, err = BoolFromStackParameter(emptyByteArrayParam)
require.NoError(t, err)
require.False(t, val)
})
t.Run("incorrect assert", func(t *testing.T) {
_, err := BoolFromStackParameter(stringParam)
require.Error(t, err)
})
}
func TestArrayFromStackParameter(t *testing.T) {
t.Run("correct assert", func(t *testing.T) {
val, err := ArrayFromStackParameter(arrayParam)
require.NoError(t, err)
require.Len(t, val, len(arrayParam.Value.([]sc.Parameter)))
})
t.Run("incorrect assert", func(t *testing.T) {
_, err := ArrayFromStackParameter(byteArrayParam)
require.Error(t, err)
})
t.Run("boolean false case", func(t *testing.T) {
val, err := ArrayFromStackParameter(falseBoolParam)
require.NoError(t, err)
require.Nil(t, val)
})
}
func TestBytesFromStackParameter(t *testing.T) {
t.Run("correct assert", func(t *testing.T) {
val, err := BytesFromStackParameter(byteArrayParam)
require.NoError(t, err)
require.Equal(t, byteArrayParam.Value.([]byte), val)
})
t.Run("incorrect assert", func(t *testing.T) {
_, err := BytesFromStackParameter(stringParam)
require.Error(t, err)
})
}
func TestIntFromStackParameter(t *testing.T) {
t.Run("correct assert", func(t *testing.T) {
val, err := IntFromStackParameter(intParam)
require.NoError(t, err)
require.Equal(t, intParam.Value.(int64), val)
val, err = IntFromStackParameter(byteWithIntParam)
require.NoError(t, err)
require.Equal(t, int64(0x0a), val)
val, err = IntFromStackParameter(emptyByteArrayParam)
require.NoError(t, err)
require.Equal(t, int64(0), val)
})
t.Run("incorrect assert", func(t *testing.T) {
_, err := IntFromStackParameter(byteArrayParam)
require.Error(t, err)
})
}
func TestStringFromStackParameter(t *testing.T) {
t.Run("correct assert", func(t *testing.T) {
val, err := StringFromStackParameter(stringParam)
require.NoError(t, err)
require.Equal(t, stringParam.Value.(string), val)
val, err = StringFromStackParameter(byteArrayParam)
require.NoError(t, err)
require.Equal(t, string(byteArrayParam.Value.([]byte)), val)
})
t.Run("incorrect assert", func(t *testing.T) {
_, err := StringFromStackParameter(intParam)
require.Error(t, err)
})
}

View file

@ -0,0 +1,151 @@
package subscriber
import (
"context"
"errors"
"sync"
"time"
"github.com/nspcc-dev/neo-go/pkg/rpc/client"
"github.com/nspcc-dev/neo-go/pkg/rpc/response"
"github.com/nspcc-dev/neo-go/pkg/rpc/response/result"
"github.com/nspcc-dev/neo-go/pkg/util"
"go.uber.org/zap"
)
type (
// Subscriber is an interface of the NotificationEvent listener.
Subscriber interface {
SubscribeForNotification(...util.Uint160) (<-chan *result.NotificationEvent, error)
UnsubscribeForNotification()
}
subscriber struct {
*sync.RWMutex
log *zap.Logger
client *client.WSClient
notify chan *result.NotificationEvent
notifyIDs map[util.Uint160]string
}
// Params is a group of Subscriber constructor parameters.
Params struct {
Log *zap.Logger
Endpoint string
DialTimeout time.Duration
}
)
var (
errNilParams = errors.New("chain/subscriber: config was not provided to the constructor")
errNilLogger = errors.New("chain/subscriber: logger was not provided to the constructor")
)
func (s *subscriber) SubscribeForNotification(contracts ...util.Uint160) (<-chan *result.NotificationEvent, error) {
s.Lock()
defer s.Unlock()
notifyIDs := make(map[util.Uint160]string, len(contracts))
for i := range contracts {
// do not subscribe to already subscribed contracts
if _, ok := s.notifyIDs[contracts[i]]; ok {
continue
}
// subscribe to contract notifications
id, err := s.client.SubscribeForExecutionNotifications(&contracts[i])
if err != nil {
// if there is some error, undo all subscriptions and return error
for _, id := range notifyIDs {
_ = s.client.Unsubscribe(id)
}
return nil, err
}
// save notification id
notifyIDs[contracts[i]] = id
}
// update global map of subscribed contracts
for contract, id := range notifyIDs {
s.notifyIDs[contract] = id
}
return s.notify, nil
}
func (s *subscriber) UnsubscribeForNotification() {
s.Lock()
defer s.Unlock()
for i := range s.notifyIDs {
err := s.client.Unsubscribe(s.notifyIDs[i])
if err != nil {
s.log.Error("unsubscribe for notification",
zap.String("event", s.notifyIDs[i]),
zap.Error(err))
}
delete(s.notifyIDs, i)
}
}
func (s *subscriber) routeNotifications(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case notification := <-s.client.Notifications:
switch notification.Type {
case response.NotificationEventID:
notification, ok := notification.Value.(*result.NotificationEvent)
if !ok {
s.log.Error("can't cast notify event to the notify struct")
continue
}
s.notify <- notification
default:
s.log.Debug("unsupported notification from the chain",
zap.Uint8("type", uint8(notification.Type)),
)
}
}
}
}
// New is a constructs Neo:Morph event listener and returns Subscriber interface.
func New(ctx context.Context, p *Params) (Subscriber, error) {
switch {
case p == nil:
return nil, errNilParams
case p.Log == nil:
return nil, errNilLogger
}
wsClient, err := client.NewWS(ctx, p.Endpoint, client.Options{
DialTimeout: p.DialTimeout,
})
if err != nil {
return nil, err
}
sub := &subscriber{
RWMutex: new(sync.RWMutex),
log: p.Log,
client: wsClient,
notify: make(chan *result.NotificationEvent),
notifyIDs: make(map[util.Uint160]string),
}
// Worker listens all events from neo-go websocket and puts them
// into corresponding channel. It may be notifications, transactions,
// new blocks. For now only notifications.
go sub.routeNotifications(ctx)
return sub, nil
}

View file

@ -0,0 +1,24 @@
package boot
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/stretchr/testify/require"
)
func TestBootstrapPeerParams(t *testing.T) {
s := BootstrapPeerParams{}
nodeInfo := &bootstrap.NodeInfo{
Address: "address",
PubKey: []byte{1, 2, 3},
Options: []string{
"opt1",
"opt2",
},
}
s.SetNodeInfo(nodeInfo)
require.Equal(t, nodeInfo, s.NodeInfo())
}

31
lib/boot/bootstrapper.go Normal file
View file

@ -0,0 +1,31 @@
package boot
import (
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/nspcc-dev/neofs-node/internal"
)
// BootstrapPeerParams is a group of parameters
// for storage node bootstrap.
type BootstrapPeerParams struct {
info *bootstrap.NodeInfo
}
// PeerBootstrapper is an interface of the NeoFS node bootstrap tool.
type PeerBootstrapper interface {
AddPeer(BootstrapPeerParams) error
}
// ErrNilPeerBootstrapper is returned by functions that expect
// a non-nil PeerBootstrapper, but received nil.
const ErrNilPeerBootstrapper = internal.Error("peer bootstrapper is nil")
// SetNodeInfo is a node info setter.
func (s *BootstrapPeerParams) SetNodeInfo(v *bootstrap.NodeInfo) {
s.info = v
}
// NodeInfo is a node info getter.
func (s BootstrapPeerParams) NodeInfo() *bootstrap.NodeInfo {
return s.info
}

46
lib/boot/storage.go Normal file
View file

@ -0,0 +1,46 @@
package boot
import (
"context"
"go.uber.org/zap"
)
// StorageBootParams is a group of parameters
// for storage node bootstrap operation.
type StorageBootParams struct {
BootstrapPeerParams
}
// StorageBootController is an entity that performs
// registration of a storage node in NeoFS network.
type StorageBootController struct {
peerBoot PeerBootstrapper
bootPrm StorageBootParams
log *zap.Logger
}
// SetPeerBootstrapper is a PeerBootstrapper setter.
func (s *StorageBootController) SetPeerBootstrapper(v PeerBootstrapper) {
s.peerBoot = v
}
// SetBootParams is a storage node bootstrap parameters setter.
func (s *StorageBootController) SetBootParams(v StorageBootParams) {
s.bootPrm = v
}
// SetLogger is a logging component setter.
func (s *StorageBootController) SetLogger(v *zap.Logger) {
s.log = v
}
// Bootstrap registers storage node in NeoFS system.
func (s StorageBootController) Bootstrap(context.Context) {
// register peer in NeoFS network
if err := s.peerBoot.AddPeer(s.bootPrm.BootstrapPeerParams); err != nil && s.log != nil {
s.log.Error("could not register storage node in network")
}
}

View file

@ -0,0 +1,109 @@
package boltdb
import (
"io/ioutil"
"log"
"os"
"path"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
"github.com/spf13/viper"
"go.etcd.io/bbolt"
)
type (
bucket struct {
db *bbolt.DB
name []byte
}
// Options groups the BoltDB bucket's options.
Options struct {
bbolt.Options
Name []byte
Path string
Perm os.FileMode
}
)
const (
defaultFilePermission = 0777
errEmptyPath = internal.Error("database empty path")
)
var _ core.Bucket = (*bucket)(nil)
func makeCopy(val []byte) []byte {
tmp := make([]byte, len(val))
copy(tmp, val)
return tmp
}
// NewOptions prepares options for badger instance.
func NewOptions(name core.BucketType, v *viper.Viper) (opts Options, err error) {
key := string(name)
opts = Options{
Options: bbolt.Options{
// set defaults:
Timeout: bbolt.DefaultOptions.Timeout,
FreelistType: bbolt.DefaultOptions.FreelistType,
// set config options:
NoSync: v.GetBool(key + ".no_sync"),
ReadOnly: v.GetBool(key + ".read_only"),
NoGrowSync: v.GetBool(key + ".no_grow_sync"),
NoFreelistSync: v.GetBool(key + ".no_freelist_sync"),
PageSize: v.GetInt(key + ".page_size"),
MmapFlags: v.GetInt(key + ".mmap_flags"),
InitialMmapSize: v.GetInt(key + ".initial_mmap_size"),
},
Name: []byte(name),
Perm: defaultFilePermission,
Path: v.GetString(key + ".path"),
}
if opts.Path == "" {
return opts, errEmptyPath
}
if tmp := v.GetDuration(key + ".lock_timeout"); tmp > 0 {
opts.Timeout = tmp
}
if perm := v.GetUint32(key + ".perm"); perm != 0 {
opts.Perm = os.FileMode(perm)
}
base := path.Dir(opts.Path)
if err := os.MkdirAll(base, opts.Perm); err != nil {
return opts, errors.Wrapf(err, "could not use `%s` dir", base)
}
return opts, nil
}
// NewBucket creates badger-bucket instance.
func NewBucket(opts *Options) (core.Bucket, error) {
log.SetOutput(ioutil.Discard) // disable default logger
db, err := bbolt.Open(opts.Path, opts.Perm, &opts.Options)
if err != nil {
return nil, err
}
err = db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(opts.Name)
return err
})
if err != nil {
return nil, err
}
return &bucket{db: db, name: opts.Name}, nil
}

View file

@ -0,0 +1,94 @@
package boltdb
import (
"os"
"github.com/mr-tron/base58"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
"go.etcd.io/bbolt"
)
// Get value by key or return error.
func (b *bucket) Get(key []byte) (data []byte, err error) {
err = b.db.View(func(txn *bbolt.Tx) error {
txn.Bucket(b.name).Cursor().Seek(key)
val := txn.Bucket(b.name).Get(key)
if val == nil {
return errors.Wrapf(core.ErrNotFound, "key=%s", base58.Encode(key))
}
data = makeCopy(val)
return nil
})
return
}
// Set value for key.
func (b *bucket) Set(key, value []byte) error {
return b.db.Update(func(txn *bbolt.Tx) error {
k, v := makeCopy(key), makeCopy(value)
return txn.Bucket(b.name).Put(k, v)
})
}
// Del removes item from bucket by key.
func (b *bucket) Del(key []byte) error {
return b.db.Update(func(txn *bbolt.Tx) error {
return txn.Bucket(b.name).Delete(key)
})
}
// Has checks key exists.
func (b *bucket) Has(key []byte) bool {
_, err := b.Get(key)
return !errors.Is(errors.Cause(err), core.ErrNotFound)
}
// Size returns size of database.
func (b *bucket) Size() int64 {
info, err := os.Stat(b.db.Path())
if err != nil {
return 0
}
return info.Size()
}
// List all items in bucket.
func (b *bucket) List() ([][]byte, error) {
var items [][]byte
if err := b.db.View(func(txn *bbolt.Tx) error {
return txn.Bucket(b.name).ForEach(func(k, _ []byte) error {
items = append(items, makeCopy(k))
return nil
})
}); err != nil {
return nil, err
}
return items, nil
}
// Filter elements by filter closure.
func (b *bucket) Iterate(handler core.FilterHandler) error {
if handler == nil {
return core.ErrNilFilterHandler
}
return b.db.View(func(txn *bbolt.Tx) error {
return txn.Bucket(b.name).ForEach(func(k, v []byte) error {
if !handler(makeCopy(k), makeCopy(v)) {
return core.ErrIteratingAborted
}
return nil
})
})
}
// Close bucket database.
func (b *bucket) Close() error {
return b.db.Close()
}

View file

@ -0,0 +1,95 @@
package boltdb
import (
"encoding/binary"
"io/ioutil"
"os"
"strings"
"testing"
"time"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
var config = strings.NewReader(`
storage:
test_bucket:
bucket: boltdb
path: ./temp/storage/test_bucket
perm: 0777
`)
func TestBucket(t *testing.T) {
file, err := ioutil.TempFile("", "test_bolt_db")
require.NoError(t, err)
require.NoError(t, file.Close())
v := viper.New()
require.NoError(t, v.ReadConfig(config))
// -- //
_, err = NewOptions("storage.test_bucket", v)
require.EqualError(t, err, errEmptyPath.Error())
v.SetDefault("storage.test_bucket.path", file.Name())
v.SetDefault("storage.test_bucket.timeout", time.Millisecond*100)
// -- //
opts, err := NewOptions("storage.test_bucket", v)
require.NoError(t, err)
db, err := NewBucket(&opts)
require.NoError(t, err)
require.NotPanics(t, func() { db.Size() })
var (
count = uint64(10)
expected = []byte("test")
)
for i := uint64(0); i < count; i++ {
key := make([]byte, 8)
binary.BigEndian.PutUint64(key, i)
require.False(t, db.Has(key))
val, err := db.Get(key)
require.EqualError(t, errors.Cause(err), core.ErrNotFound.Error())
require.Empty(t, val)
require.NoError(t, db.Set(key, expected))
require.True(t, db.Has(key))
val, err = db.Get(key)
require.NoError(t, err)
require.Equal(t, expected, val)
keys, err := db.List()
require.NoError(t, err)
require.Len(t, keys, 1)
require.Equal(t, key, keys[0])
require.EqualError(t, db.Iterate(nil), core.ErrNilFilterHandler.Error())
items, err := core.ListBucketItems(db, func(_, _ []byte) bool { return true })
require.NoError(t, err)
require.Len(t, items, 1)
require.Equal(t, key, items[0].Key)
require.Equal(t, val, items[0].Val)
require.NoError(t, db.Del(key))
require.False(t, db.Has(key))
val, err = db.Get(key)
require.EqualError(t, errors.Cause(err), core.ErrNotFound.Error())
require.Empty(t, val)
}
require.NoError(t, db.Close())
require.NoError(t, os.RemoveAll(file.Name()))
}

View file

@ -0,0 +1,25 @@
package main
import (
"github.com/nspcc-dev/neofs-node/lib/buckets/boltdb"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
"github.com/spf13/viper"
)
var _ = PrepareBucket
// PrepareBucket is interface method for bucket.
func PrepareBucket(name core.BucketType, v *viper.Viper) (db core.Bucket, err error) {
var opts boltdb.Options
if opts, err = boltdb.NewOptions("storage."+name, v); err != nil {
err = errors.Wrapf(err, "%q: could not prepare options", name)
return
} else if db, err = boltdb.NewBucket(&opts); err != nil {
err = errors.Wrapf(err, "%q: could not prepare bucket", name)
return
}
return
}

View file

@ -0,0 +1,101 @@
package fsbucket
import (
"os"
"github.com/mr-tron/base58"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
"github.com/spf13/viper"
"go.uber.org/atomic"
)
type (
bucket struct {
dir string
perm os.FileMode
}
treeBucket struct {
dir string
perm os.FileMode
depth int
prefixLength int
sz *atomic.Int64
}
)
const (
defaultDirectory = "fsbucket"
defaultPermissions = 0755
defaultDepth = 2
defaultPrefixLen = 2
)
const errShortKey = internal.Error("key is too short for tree fs bucket")
var _ core.Bucket = (*bucket)(nil)
func stringifyKey(key []byte) string {
return base58.Encode(key)
}
func decodeKey(key string) []byte {
k, err := base58.Decode(key)
if err != nil {
panic(err) // it can fail only for not base58 strings
}
return k
}
// NewBucket creates new in-memory bucket instance.
func NewBucket(name core.BucketType, v *viper.Viper) (core.Bucket, error) {
var (
key = "storage." + string(name)
dir string
perm os.FileMode
prefixLen int
depth int
)
if dir = v.GetString(key + ".directory"); dir == "" {
dir = defaultDirectory
}
if perm = os.FileMode(v.GetInt(key + ".permissions")); perm == 0 {
perm = defaultPermissions
}
if depth = v.GetInt(key + ".depth"); depth <= 0 {
depth = defaultDepth
}
if prefixLen = v.GetInt(key + ".prefix_len"); prefixLen <= 0 {
prefixLen = defaultPrefixLen
}
if err := os.MkdirAll(dir, perm); err != nil {
return nil, errors.Wrapf(err, "could not create bucket %s", string(name))
}
if v.GetBool(key + ".tree_enabled") {
b := &treeBucket{
dir: dir,
perm: perm,
depth: depth,
prefixLength: prefixLen,
}
b.sz = atomic.NewInt64(b.size())
return b, nil
}
return &bucket{
dir: dir,
perm: perm,
}, nil
}

View file

@ -0,0 +1,107 @@
package fsbucket
import (
"io/ioutil"
"os"
"path"
"path/filepath"
"github.com/nspcc-dev/neofs-node/lib/core"
)
// Get value by key.
func (b *bucket) Get(key []byte) ([]byte, error) {
p := path.Join(b.dir, stringifyKey(key))
if _, err := os.Stat(p); os.IsNotExist(err) {
return nil, core.ErrNotFound
}
return ioutil.ReadFile(p)
}
// Set value by key.
func (b *bucket) Set(key, value []byte) error {
p := path.Join(b.dir, stringifyKey(key))
return ioutil.WriteFile(p, value, b.perm)
}
// Del value by key.
func (b *bucket) Del(key []byte) error {
p := path.Join(b.dir, stringifyKey(key))
if _, err := os.Stat(p); os.IsNotExist(err) {
return core.ErrNotFound
}
return os.Remove(p)
}
// Has checks key exists.
func (b *bucket) Has(key []byte) bool {
p := path.Join(b.dir, stringifyKey(key))
_, err := os.Stat(p)
return err == nil
}
func listing(root string, fn func(path string, info os.FileInfo) error) error {
return filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
if fn == nil {
return nil
}
return fn(p, info)
})
}
// Size of bucket.
func (b *bucket) Size() (size int64) {
err := listing(b.dir, func(_ string, info os.FileInfo) error {
size += info.Size()
return nil
})
if err != nil {
size = 0
}
return
}
// List all bucket items.
func (b *bucket) List() ([][]byte, error) {
buckets := make([][]byte, 0)
err := listing(b.dir, func(p string, info os.FileInfo) error {
buckets = append(buckets, decodeKey(info.Name()))
return nil
})
return buckets, err
}
// Filter bucket items by closure.
func (b *bucket) Iterate(handler core.FilterHandler) error {
return listing(b.dir, func(p string, info os.FileInfo) error {
key := decodeKey(info.Name())
val, err := ioutil.ReadFile(p)
if err != nil {
return err
}
if !handler(key, val) {
return core.ErrIteratingAborted
}
return nil
})
}
// Close bucket (just empty).
func (b *bucket) Close() error {
return os.RemoveAll(b.dir)
}

View file

@ -0,0 +1,44 @@
package fsbucket
import "sync"
type (
queue struct {
*sync.RWMutex
buf []elem
}
elem struct {
depth int
prefix string
path string
}
)
func newQueue(n int) *queue {
return &queue{
RWMutex: new(sync.RWMutex),
buf: make([]elem, 0, n),
}
}
func (q *queue) Len() int {
return len(q.buf)
}
func (q *queue) Push(s elem) {
q.Lock()
q.buf = append(q.buf, s)
q.Unlock()
}
func (q *queue) Pop() (s elem) {
q.Lock()
if len(q.buf) > 0 {
s = q.buf[0]
q.buf = q.buf[1:]
}
q.Unlock()
return
}

View file

@ -0,0 +1,261 @@
package fsbucket
import (
"encoding/hex"
"io/ioutil"
"os"
"path"
"strings"
"github.com/nspcc-dev/neofs-node/lib/core"
)
const queueCap = 1000
func stringifyHexKey(key []byte) string {
return hex.EncodeToString(key)
}
func decodeHexKey(key string) ([]byte, error) {
k, err := hex.DecodeString(key)
if err != nil {
return nil, err
}
return k, nil
}
// treePath returns slice of the dir names that contain the path
// and filename, e.g. 0xabcdef => []string{"ab", "cd"}, "abcdef".
// In case of errors - return nil slice.
func (b *treeBucket) treePath(key []byte) ([]string, string) {
filename := stringifyHexKey(key)
if len(filename) <= b.prefixLength*b.depth {
return nil, filename
}
filepath := filename
dirs := make([]string, 0, b.depth)
for i := 0; i < b.depth; i++ {
dirs = append(dirs, filepath[:b.prefixLength])
filepath = filepath[b.prefixLength:]
}
return dirs, filename
}
// Get value by key.
func (b *treeBucket) Get(key []byte) ([]byte, error) {
dirPaths, filename := b.treePath(key)
if dirPaths == nil {
return nil, errShortKey
}
p := path.Join(b.dir, path.Join(dirPaths...), filename)
if _, err := os.Stat(p); os.IsNotExist(err) {
return nil, core.ErrNotFound
}
return ioutil.ReadFile(p)
}
// Set value by key.
func (b *treeBucket) Set(key, value []byte) error {
dirPaths, filename := b.treePath(key)
if dirPaths == nil {
return errShortKey
}
var (
dirPath = path.Join(dirPaths...)
p = path.Join(b.dir, dirPath, filename)
)
if err := os.MkdirAll(path.Join(b.dir, dirPath), b.perm); err != nil {
return err
}
err := ioutil.WriteFile(p, value, b.perm)
if err == nil {
b.sz.Add(int64(len(value)))
}
return err
}
// Del value by key.
func (b *treeBucket) Del(key []byte) error {
dirPaths, filename := b.treePath(key)
if dirPaths == nil {
return errShortKey
}
var (
err error
fi os.FileInfo
p = path.Join(b.dir, path.Join(dirPaths...), filename)
)
if fi, err = os.Stat(p); os.IsNotExist(err) {
return core.ErrNotFound
} else if err = os.Remove(p); err == nil {
b.sz.Sub(fi.Size())
}
return err
}
// Has checks if key exists.
func (b *treeBucket) Has(key []byte) bool {
dirPaths, filename := b.treePath(key)
if dirPaths == nil {
return false
}
p := path.Join(b.dir, path.Join(dirPaths...), filename)
_, err := os.Stat(p)
return err == nil
}
// There might be two implementation of listing method: simple with `filepath.Walk()`
// or more complex implementation with path checks, BFS etc. `filepath.Walk()` might
// be slow in large dirs due to sorting operations and non controllable depth.
func (b *treeBucket) listing(root string, fn func(path string, info os.FileInfo) error) error {
// todo: DFS might be better since it won't store many files in queue.
// todo: queue length can be specified as a parameter
q := newQueue(queueCap)
q.Push(elem{path: root})
for q.Len() > 0 {
e := q.Pop()
s, err := os.Lstat(e.path)
if err != nil {
// might be better to log and ignore
return err
}
// check if it is correct file
if !s.IsDir() {
// we accept files that located in excepted depth and have correct prefix
// e.g. file 'abcdef0123' => /ab/cd/abcdef0123
if e.depth == b.depth+1 && strings.HasPrefix(s.Name(), e.prefix) {
err = fn(e.path, s)
if err != nil {
// might be better to log and ignore
return err
}
}
continue
}
// ignore dirs with inappropriate length or depth
if e.depth > b.depth || (e.depth > 0 && len(s.Name()) > b.prefixLength) {
continue
}
files, err := readDirNames(e.path)
if err != nil {
// might be better to log and ignore
return err
}
for i := range files {
// add prefix of all dirs in path except root dir
var prefix string
if e.depth > 0 {
prefix = e.prefix + s.Name()
}
q.Push(elem{
depth: e.depth + 1,
prefix: prefix,
path: path.Join(e.path, files[i]),
})
}
}
return nil
}
// Size returns the size of the bucket in bytes.
func (b *treeBucket) Size() int64 {
return b.sz.Load()
}
func (b *treeBucket) size() (size int64) {
err := b.listing(b.dir, func(_ string, info os.FileInfo) error {
size += info.Size()
return nil
})
if err != nil {
size = 0
}
return
}
// List all bucket items.
func (b *treeBucket) List() ([][]byte, error) {
buckets := make([][]byte, 0)
err := b.listing(b.dir, func(p string, info os.FileInfo) error {
key, err := decodeHexKey(info.Name())
if err != nil {
return err
}
buckets = append(buckets, key)
return nil
})
return buckets, err
}
// Filter bucket items by closure.
func (b *treeBucket) Iterate(handler core.FilterHandler) error {
return b.listing(b.dir, func(p string, info os.FileInfo) error {
val, err := ioutil.ReadFile(path.Join(b.dir, p))
if err != nil {
return err
}
key, err := decodeHexKey(info.Name())
if err != nil {
return err
}
if !handler(key, val) {
return core.ErrIteratingAborted
}
return nil
})
}
// Close bucket (remove all available data).
func (b *treeBucket) Close() error {
return os.RemoveAll(b.dir)
}
// readDirNames copies `filepath.readDirNames()` without sorting the output.
func readDirNames(dirname string) ([]string, error) {
f, err := os.Open(dirname)
if err != nil {
return nil, err
}
names, err := f.Readdirnames(-1)
if err != nil {
return nil, err
}
f.Close()
return names, nil
}

View file

@ -0,0 +1,324 @@
package fsbucket
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
"github.com/nspcc-dev/neofs-node/lib/core"
)
func prepareTree(badFiles bool) (string, error) {
name := make([]byte, 32)
root, err := ioutil.TempDir("", "treeBucket_test")
if err != nil {
return "", err
}
// paths must contain strings with hex ascii symbols
paths := [][]string{
{root, "abcd"},
{root, "abcd", "cdef"},
{root, "abcd", "cd01"},
{root, "0123", "2345"},
{root, "0123", "2345", "4567"},
}
dirs := make([]string, len(paths))
for i := range paths {
dirs[i] = path.Join(paths[i]...)
err = os.MkdirAll(dirs[i], 0700)
if err != nil {
return "", err
}
// create couple correct files
for j := 0; j < 2; j++ {
_, err := rand.Read(name)
if err != nil {
return "", err
}
filePrefix := new(strings.Builder)
for k := 1; k < len(paths[i]); k++ {
filePrefix.WriteString(paths[i][k])
}
filePrefix.WriteString(hex.EncodeToString(name))
file, err := os.OpenFile(path.Join(dirs[i], filePrefix.String()), os.O_CREATE, 0700)
if err != nil {
return "", err
}
file.Close()
}
if !badFiles {
continue
}
// create one bad file
_, err := rand.Read(name)
if err != nil {
return "", err
}
file, err := os.OpenFile(path.Join(dirs[i], "fff"+hex.EncodeToString(name)), os.O_CREATE, 0700)
if err != nil {
return "", err
}
file.Close()
}
return root, nil
}
func TestTreebucket_List(t *testing.T) {
root, err := prepareTree(true)
require.NoError(t, err)
defer os.RemoveAll(root)
b := treeBucket{
dir: root,
perm: 0700,
depth: 1,
prefixLength: 4,
}
results, err := b.List()
require.NoError(t, err)
require.Len(t, results, 2)
b.depth = 2
results, err = b.List()
require.NoError(t, err)
require.Len(t, results, 6)
b.depth = 3
results, err = b.List()
require.NoError(t, err)
require.Len(t, results, 2)
b.depth = 4
results, err = b.List()
require.NoError(t, err)
require.Len(t, results, 0)
}
func TestTreebucket(t *testing.T) {
root, err := prepareTree(true)
require.NoError(t, err)
defer os.RemoveAll(root)
b := treeBucket{
dir: root,
perm: 0700,
depth: 2,
prefixLength: 4,
sz: atomic.NewInt64(0),
}
results, err := b.List()
require.NoError(t, err)
require.Len(t, results, 6)
t.Run("Get", func(t *testing.T) {
for i := range results {
_, err = b.Get(results[i])
require.NoError(t, err)
}
_, err = b.Get([]byte("Hello world!"))
require.Error(t, err)
})
t.Run("Has", func(t *testing.T) {
for i := range results {
require.True(t, b.Has(results[i]))
}
require.False(t, b.Has([]byte("Unknown key")))
})
t.Run("Set", func(t *testing.T) {
keyHash := sha256.Sum256([]byte("Set this key"))
key := keyHash[:]
value := make([]byte, 32)
rand.Read(value)
// set sha256 key
err := b.Set(key, value)
require.NoError(t, err)
require.True(t, b.Has(key))
data, err := b.Get(key)
require.NoError(t, err)
require.Equal(t, data, value)
filename := hex.EncodeToString(key)
_, err = os.Lstat(path.Join(root, filename[:4], filename[4:8], filename))
require.NoError(t, err)
// set key that cannot be placed in the required dir depth
key, err = hex.DecodeString("abcdef")
require.NoError(t, err)
err = b.Set(key, value)
require.Error(t, err)
})
t.Run("Delete", func(t *testing.T) {
keyHash := sha256.Sum256([]byte("Delete this key"))
key := keyHash[:]
value := make([]byte, 32)
rand.Read(value)
err := b.Set(key, value)
require.NoError(t, err)
// delete sha256 key
err = b.Del(key)
require.NoError(t, err)
_, err = b.Get(key)
require.Error(t, err)
filename := hex.EncodeToString(key)
_, err = os.Lstat(path.Join(root, filename[:4], filename[4:8], filename))
require.Error(t, err)
})
}
func TestTreebucket_Close(t *testing.T) {
root, err := prepareTree(true)
require.NoError(t, err)
defer os.RemoveAll(root)
b := treeBucket{
dir: root,
perm: 0700,
depth: 2,
prefixLength: 4,
}
err = b.Close()
require.NoError(t, err)
_, err = os.Lstat(root)
require.Error(t, err)
}
func TestTreebucket_Size(t *testing.T) {
root, err := prepareTree(true)
require.NoError(t, err)
defer os.RemoveAll(root)
var size int64 = 1024
key := []byte("Set this key")
value := make([]byte, size)
rand.Read(value)
b := treeBucket{
dir: root,
perm: 0700,
depth: 2,
prefixLength: 4,
sz: atomic.NewInt64(0),
}
err = b.Set(key, value)
require.NoError(t, err)
require.Equal(t, size, b.Size())
}
func BenchmarkTreebucket_List(b *testing.B) {
root, err := prepareTree(false)
defer os.RemoveAll(root)
if err != nil {
b.Error(err)
}
treeFSBucket := &treeBucket{
dir: root,
perm: 0755,
depth: 2,
prefixLength: 4,
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := treeFSBucket.List()
if err != nil {
b.Error(err)
}
}
}
func BenchmarkFilewalkBucket_List(b *testing.B) {
root, err := prepareTree(false)
defer os.RemoveAll(root)
if err != nil {
b.Error(err)
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
buckets := make([]core.BucketItem, 0)
filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return nil
}
val, err := ioutil.ReadFile(path)
if err != nil {
return err
}
key, err := decodeHexKey(info.Name())
if err != nil {
return err
}
buckets = append(buckets, core.BucketItem{
Key: key,
Val: val,
})
return nil
})
}
}
func BenchmarkTreeBucket_Size(b *testing.B) {
root, err := prepareTree(false)
defer os.RemoveAll(root)
if err != nil {
b.Error(err)
}
treeFSBucket := &treeBucket{
dir: root,
perm: 0755,
depth: 2,
prefixLength: 4,
}
treeFSBucket.sz = atomic.NewInt64(treeFSBucket.size())
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = treeFSBucket.Size()
}
}

64
lib/buckets/init.go Normal file
View file

@ -0,0 +1,64 @@
package buckets
import (
"plugin"
"strings"
"github.com/nspcc-dev/neofs-node/lib/buckets/boltdb"
"github.com/nspcc-dev/neofs-node/lib/buckets/fsbucket"
"github.com/nspcc-dev/neofs-node/lib/buckets/inmemory"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
"github.com/spf13/viper"
"go.uber.org/zap"
)
const (
// BoltDBBucket is a name of BoltDB bucket.
BoltDBBucket = "boltdb"
// InMemoryBucket is a name RAM bucket.
InMemoryBucket = "in-memory"
// FileSystemBucket is a name of file system bucket.
FileSystemBucket = "fsbucket"
bucketSymbol = "PrepareBucket"
)
// NewBucket is a bucket's constructor.
func NewBucket(name core.BucketType, l *zap.Logger, v *viper.Viper) (core.Bucket, error) {
bucket := v.GetString("storage." + string(name) + ".bucket")
l.Info("initialize bucket",
zap.String("name", string(name)),
zap.String("bucket", bucket))
switch strings.ToLower(bucket) {
case FileSystemBucket:
return fsbucket.NewBucket(name, v)
case InMemoryBucket:
return inmemory.NewBucket(name, v), nil
case BoltDBBucket:
opts, err := boltdb.NewOptions("storage."+name, v)
if err != nil {
return nil, err
}
return boltdb.NewBucket(&opts)
default:
instance, err := plugin.Open(bucket)
if err != nil {
return nil, errors.Wrapf(err, "could not load bucket: `%s`", bucket)
}
sym, err := instance.Lookup(bucketSymbol)
if err != nil {
return nil, errors.Wrapf(err, "could not find bucket signature: `%s`", bucket)
}
return sym.(func(core.BucketType, *viper.Viper) (core.Bucket, error))(name, v)
}
}

View file

@ -0,0 +1,60 @@
package inmemory
import (
"sync"
"github.com/mr-tron/base58"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/spf13/viper"
)
type (
bucket struct {
*sync.RWMutex
items map[string][]byte
}
)
const (
defaultCapacity = 100
)
var (
_ core.Bucket = (*bucket)(nil)
// for in usage
_ = NewBucket
)
func stringifyKey(key []byte) string {
return base58.Encode(key)
}
func decodeKey(key string) []byte {
k, err := base58.Decode(key)
if err != nil {
panic(err) // it can fail only for not base58 strings
}
return k
}
func makeCopy(val []byte) []byte {
tmp := make([]byte, len(val))
copy(tmp, val)
return tmp
}
// NewBucket creates new in-memory bucket instance.
func NewBucket(name core.BucketType, v *viper.Viper) core.Bucket {
var capacity int
if capacity = v.GetInt("storage." + string(name) + ".capacity"); capacity <= 0 {
capacity = defaultCapacity
}
return &bucket{
RWMutex: new(sync.RWMutex),
items: make(map[string][]byte, capacity),
}
}

View file

@ -0,0 +1,107 @@
package inmemory
import (
"unsafe"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
)
// Get value by key.
func (b *bucket) Get(key []byte) ([]byte, error) {
k := stringifyKey(key)
b.RLock()
val, ok := b.items[k]
result := makeCopy(val)
b.RUnlock()
if !ok {
return nil, errors.Wrapf(core.ErrNotFound, "key=`%s`", k)
}
return result, nil
}
// Set value by key.
func (b *bucket) Set(key, value []byte) error {
k := stringifyKey(key)
b.Lock()
b.items[k] = makeCopy(value)
b.Unlock()
return nil
}
// Del value by key.
func (b *bucket) Del(key []byte) error {
k := stringifyKey(key)
b.Lock()
delete(b.items, k)
b.Unlock()
return nil
}
// Has checks key exists.
func (b *bucket) Has(key []byte) bool {
k := stringifyKey(key)
b.RLock()
_, ok := b.items[k]
b.RUnlock()
return ok
}
// Size size of bucket.
func (b *bucket) Size() int64 {
b.RLock()
// TODO we must replace in future
size := unsafe.Sizeof(b.items)
b.RUnlock()
return int64(size)
}
func (b *bucket) List() ([][]byte, error) {
var result = make([][]byte, 0)
b.RLock()
for key := range b.items {
result = append(result, decodeKey(key))
}
b.RUnlock()
return result, nil
}
// Filter items by closure.
func (b *bucket) Iterate(handler core.FilterHandler) error {
if handler == nil {
return core.ErrNilFilterHandler
}
b.RLock()
for key, val := range b.items {
k, v := decodeKey(key), makeCopy(val)
if !handler(k, v) {
return core.ErrIteratingAborted
}
}
b.RUnlock()
return nil
}
// Close bucket (just empty).
func (b *bucket) Close() error {
b.Lock()
b.items = make(map[string][]byte)
b.Unlock()
return nil
}

15
lib/container/alias.go Normal file
View file

@ -0,0 +1,15 @@
package container
import (
"github.com/nspcc-dev/neofs-api-go/container"
"github.com/nspcc-dev/neofs-api-go/refs"
)
// Container is a type alias of Container.
type Container = container.Container
// CID is a type alias of CID.
type CID = refs.CID
// OwnerID is a type alias of OwnerID.
type OwnerID = refs.OwnerID

134
lib/container/storage.go Normal file
View file

@ -0,0 +1,134 @@
package container
import (
"context"
)
// GetParams is a group of parameters for container receiving operation.
type GetParams struct {
ctxValue
cidValue
}
// GetResult is a group of values returned by container receiving operation.
type GetResult struct {
cnrValue
}
// PutParams is a group of parameters for container storing operation.
type PutParams struct {
ctxValue
cnrValue
}
// PutResult is a group of values returned by container storing operation.
type PutResult struct {
cidValue
}
// DeleteParams is a group of parameters for container removal operation.
type DeleteParams struct {
ctxValue
cidValue
ownerID OwnerID
}
// DeleteResult is a group of values returned by container removal operation.
type DeleteResult struct{}
// ListParams is a group of parameters for container listing operation.
type ListParams struct {
ctxValue
ownerIDList []OwnerID
}
// ListResult is a group of values returned by container listing operation.
type ListResult struct {
cidList []CID
}
type cnrValue struct {
cnr *Container
}
type cidValue struct {
cid CID
}
type ctxValue struct {
ctx context.Context
}
// Storage is an interface of the storage of NeoFS containers.
type Storage interface {
GetContainer(GetParams) (*GetResult, error)
PutContainer(PutParams) (*PutResult, error)
DeleteContainer(DeleteParams) (*DeleteResult, error)
ListContainers(ListParams) (*ListResult, error)
// TODO: add EACL methods
}
// Context is a context getter.
func (s ctxValue) Context() context.Context {
return s.ctx
}
// SetContext is a context setter.
func (s *ctxValue) SetContext(v context.Context) {
s.ctx = v
}
// CID is a container ID getter.
func (s cidValue) CID() CID {
return s.cid
}
// SetCID is a container ID getter.
func (s *cidValue) SetCID(v CID) {
s.cid = v
}
// Container is a container getter.
func (s cnrValue) Container() *Container {
return s.cnr
}
// SetContainer is a container setter.
func (s *cnrValue) SetContainer(v *Container) {
s.cnr = v
}
// OwnerID is an owner ID getter.
func (s DeleteParams) OwnerID() OwnerID {
return s.ownerID
}
// SetOwnerID is an owner ID setter.
func (s *DeleteParams) SetOwnerID(v OwnerID) {
s.ownerID = v
}
// OwnerIDList is an owner ID list getter.
func (s ListParams) OwnerIDList() []OwnerID {
return s.ownerIDList
}
// SetOwnerIDList is an owner ID list setter.
func (s *ListParams) SetOwnerIDList(v ...OwnerID) {
s.ownerIDList = v
}
// CIDList is a container ID list getter.
func (s ListResult) CIDList() []CID {
return s.cidList
}
// SetCIDList is a container ID list setter.
func (s *ListResult) SetCIDList(v []CID) {
s.cidList = v
}

View file

@ -0,0 +1,83 @@
package container
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGetParams(t *testing.T) {
p := new(GetParams)
cid := CID{1, 2, 3}
p.SetCID(cid)
require.Equal(t, cid, p.CID())
}
func TestGetResult(t *testing.T) {
r := new(GetResult)
cnr := &Container{
OwnerID: OwnerID{1, 2, 3},
}
r.SetContainer(cnr)
require.Equal(t, cnr, r.Container())
}
func TestPutParams(t *testing.T) {
p := new(PutParams)
cnr := &Container{
OwnerID: OwnerID{1, 2, 3},
}
p.SetContainer(cnr)
require.Equal(t, cnr, p.Container())
}
func TestPutResult(t *testing.T) {
r := new(PutResult)
cid := CID{1, 2, 3}
r.SetCID(cid)
require.Equal(t, cid, r.CID())
}
func TestDeleteParams(t *testing.T) {
p := new(DeleteParams)
ownerID := OwnerID{1, 2, 3}
p.SetOwnerID(ownerID)
require.Equal(t, ownerID, p.OwnerID())
cid := CID{4, 5, 6}
p.SetCID(cid)
require.Equal(t, cid, p.CID())
}
func TestListParams(t *testing.T) {
p := new(ListParams)
ownerIDList := []OwnerID{
{1, 2, 3},
{4, 5, 6},
}
p.SetOwnerIDList(ownerIDList...)
require.Equal(t, ownerIDList, p.OwnerIDList())
}
func TestListResult(t *testing.T) {
r := new(ListResult)
cidList := []CID{
{1, 2, 3},
{4, 5, 6},
}
r.SetCIDList(cidList)
require.Equal(t, cidList, r.CIDList())
}

94
lib/core/storage.go Normal file
View file

@ -0,0 +1,94 @@
package core
import (
"github.com/nspcc-dev/neofs-node/internal"
"github.com/pkg/errors"
)
type (
// BucketType is name of bucket
BucketType string
// FilterHandler where you receive key/val in your closure
FilterHandler func(key, val []byte) bool
// BucketItem used in filter
BucketItem struct {
Key []byte
Val []byte
}
// Bucket is sub-store interface
Bucket interface {
Get(key []byte) ([]byte, error)
Set(key, value []byte) error
Del(key []byte) error
Has(key []byte) bool
Size() int64
List() ([][]byte, error)
Iterate(FilterHandler) error
// Steam can be implemented by badger.Stream, but not for now
// Stream(ctx context.Context, key []byte, cb func(io.ReadWriter) error) error
Close() error
}
// Storage component interface
Storage interface {
GetBucket(name BucketType) (Bucket, error)
Size() int64
Close() error
}
)
const (
// BlobStore is a blob bucket name.
BlobStore BucketType = "blob"
// MetaStore is a meta bucket name.
MetaStore BucketType = "meta"
// SpaceMetricsStore is a space metrics bucket name.
SpaceMetricsStore BucketType = "space-metrics"
)
var (
// ErrNilFilterHandler when FilterHandler is empty
ErrNilFilterHandler = errors.New("handler can't be nil")
// ErrNotFound is returned by key-value storage methods
// that could not find element by key.
ErrNotFound = internal.Error("key not found")
)
// ErrIteratingAborted is returned by storage iterator
// after iteration has been interrupted.
var ErrIteratingAborted = errors.New("iteration aborted")
var errEmptyBucket = errors.New("empty bucket")
func (t BucketType) String() string { return string(t) }
// ListBucketItems performs iteration over Bucket and returns the full list of its items.
func ListBucketItems(b Bucket, h FilterHandler) ([]BucketItem, error) {
if b == nil {
return nil, errEmptyBucket
} else if h == nil {
return nil, ErrNilFilterHandler
}
items := make([]BucketItem, 0)
if err := b.Iterate(func(key, val []byte) bool {
if h(key, val) {
items = append(items, BucketItem{
Key: key,
Val: val,
})
}
return true
}); err != nil {
return nil, err
}
return items, nil
}

65
lib/core/storage_test.go Normal file
View file

@ -0,0 +1,65 @@
package core
import (
"crypto/rand"
"testing"
"github.com/stretchr/testify/require"
)
type testBucket struct {
Bucket
items []BucketItem
}
func (s *testBucket) Iterate(f FilterHandler) error {
for i := range s.items {
if !f(s.items[i].Key, s.items[i].Val) {
return ErrIteratingAborted
}
}
return nil
}
func TestListBucketItems(t *testing.T) {
_, err := ListBucketItems(nil, nil)
require.EqualError(t, err, errEmptyBucket.Error())
b := new(testBucket)
_, err = ListBucketItems(b, nil)
require.EqualError(t, err, ErrNilFilterHandler.Error())
var (
count = 10
ln = 10
items = make([]BucketItem, 0, count)
)
for i := 0; i < count; i++ {
items = append(items, BucketItem{
Key: testData(t, ln),
Val: testData(t, ln),
})
}
b.items = items
res, err := ListBucketItems(b, func(key, val []byte) bool { return true })
require.NoError(t, err)
require.Equal(t, items, res)
res, err = ListBucketItems(b, func(key, val []byte) bool { return false })
require.NoError(t, err)
require.Empty(t, res)
}
func testData(t *testing.T, sz int) []byte {
d := make([]byte, sz)
_, err := rand.Read(d)
require.NoError(t, err)
return d
}

22
lib/core/validator.go Normal file
View file

@ -0,0 +1,22 @@
package core
import (
"github.com/nspcc-dev/neofs-api-go/service"
"github.com/nspcc-dev/neofs-node/internal"
)
// ErrMissingKeySignPairs is returned by functions that expect
// a non-empty SignKeyPair slice, but received empty.
const ErrMissingKeySignPairs = internal.Error("missing key-signature pairs")
// VerifyRequestWithSignatures checks if request has signatures and all of them are valid.
//
// Returns ErrMissingKeySignPairs if request does not have signatures.
// Otherwise, behaves like service.VerifyRequestData.
func VerifyRequestWithSignatures(req service.RequestVerifyData) error {
if len(req.GetSignKeyPairs()) == 0 {
return ErrMissingKeySignPairs
}
return service.VerifyRequestData(req)
}

69
lib/core/verify.go Normal file
View file

@ -0,0 +1,69 @@
package core
import (
"context"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-api-go/service"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/nspcc-dev/neofs-node/internal"
)
// OwnerKeyContainer is an interface of the container of owner's ID and key pair with read access.
type OwnerKeyContainer interface {
GetOwnerID() refs.OwnerID
GetOwnerKey() []byte
}
// OwnerKeyVerifier is an interface of OwnerKeyContainer validator.
type OwnerKeyVerifier interface {
// Must check if OwnerKeyContainer satisfies a certain criterion.
// Nil error is equivalent to matching the criterion.
VerifyKey(context.Context, OwnerKeyContainer) error
}
type neoKeyVerifier struct{}
// ErrNilOwnerKeyContainer is returned by functions that expect a non-nil
// OwnerKeyContainer, but received nil.
const ErrNilOwnerKeyContainer = internal.Error("owner-key container is nil")
// ErrNilOwnerKeyVerifier is returned by functions that expect a non-nil
// OwnerKeyVerifier, but received nil.
const ErrNilOwnerKeyVerifier = internal.Error("owner-key verifier is nil")
// NewNeoKeyVerifier creates a new Neo owner key verifier and return a OwnerKeyVerifier interface.
func NewNeoKeyVerifier() OwnerKeyVerifier {
return new(neoKeyVerifier)
}
// VerifyKey checks if the public key converts to owner ID.
//
// If passed OwnerKeyContainer is nil, ErrNilOwnerKeyContainer returns.
// If public key cannot be unmarshaled, service.ErrInvalidPublicKeyBytes returns.
// If public key is not converted to owner ID, service.ErrWrongOwner returns.
// With neo:morph adoption public key can be unrelated to owner ID. In this
// case VerifyKey should call NeoFS.ID smart-contract to check whether public
// key is bounded with owner ID. If there is no bound, then return
// service.ErrWrongOwner.
func (s neoKeyVerifier) VerifyKey(_ context.Context, src OwnerKeyContainer) error {
if src == nil {
return ErrNilOwnerKeyContainer
}
pubKey := crypto.UnmarshalPublicKey(src.GetOwnerKey())
if pubKey == nil {
return service.ErrInvalidPublicKeyBytes
}
ownerFromKey, err := refs.NewOwnerID(pubKey)
if err != nil {
return err
}
if !ownerFromKey.Equal(src.GetOwnerID()) {
return service.ErrWrongOwner
}
return nil
}

59
lib/fix/catch.go Normal file
View file

@ -0,0 +1,59 @@
package fix
import (
"fmt"
"reflect"
"go.uber.org/zap"
)
func (a *app) Catch(err error) {
if err == nil {
return
}
if a.log == nil {
panic(err)
}
a.log.Fatal("Can't run app",
zap.Error(err))
}
// CatchTrace catch errors for debugging
// use that function just for debug your application.
func (a *app) CatchTrace(err error) {
if err == nil {
return
}
// digging into the root of the problem
for {
var (
ok bool
v = reflect.ValueOf(err)
fn reflect.Value
)
if v.Type().Kind() != reflect.Struct {
break
}
if !v.FieldByName("Reason").IsValid() {
break
}
if v.FieldByName("Func").IsValid() {
fn = v.FieldByName("Func")
}
fmt.Printf("Place: %#v\nReason: %s\n\n", fn, err)
if err, ok = v.FieldByName("Reason").Interface().(error); !ok {
err = v.Interface().(error)
break
}
}
panic(err)
}

53
lib/fix/config/config.go Normal file
View file

@ -0,0 +1,53 @@
package config
import (
"strings"
"github.com/spf13/viper"
)
// Params groups the parameters of configuration.
type Params struct {
File string
Type string
Prefix string
Name string
Version string
AppDefaults func(v *viper.Viper)
}
// NewConfig is a configuration tool's constructor.
func NewConfig(p Params) (v *viper.Viper, err error) {
v = viper.New()
v.SetEnvPrefix(p.Prefix)
v.AutomaticEnv()
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
v.SetDefault("app.name", p.Name)
v.SetDefault("app.version", p.Version)
if p.AppDefaults != nil {
p.AppDefaults(v)
}
if p.fromFile() {
v.SetConfigFile(p.File)
v.SetConfigType(p.safeType())
err = v.ReadInConfig()
}
return v, err
}
func (p Params) fromFile() bool {
return p.File != ""
}
func (p Params) safeType() string {
if p.Type == "" {
p.Type = "yaml"
}
return strings.ToLower(p.Type)
}

112
lib/fix/fix.go Normal file
View file

@ -0,0 +1,112 @@
package fix
import (
"context"
"fmt"
"strconv"
"strings"
"github.com/nspcc-dev/neofs-node/lib/fix/config"
"github.com/nspcc-dev/neofs-node/lib/fix/logger"
"github.com/nspcc-dev/neofs-node/lib/fix/module"
"github.com/nspcc-dev/neofs-node/misc"
"github.com/pkg/errors"
"github.com/spf13/viper"
"go.uber.org/dig"
"go.uber.org/zap"
)
type (
// App is an interface of executable application.
App interface {
Run() error
RunAndCatch()
}
app struct {
err error
log *zap.Logger
di *dig.Container
runner interface{}
}
// Settings groups the application parameters.
Settings struct {
File string
Type string
Name string
Prefix string
Build string
Version string
Runner interface{}
AppDefaults func(v *viper.Viper)
}
)
func (a *app) RunAndCatch() {
err := a.Run()
if errors.Is(err, context.Canceled) {
return
}
if ok, _ := strconv.ParseBool(misc.Debug); ok {
a.CatchTrace(err)
}
a.Catch(err)
}
func (a *app) Run() error {
if a.err != nil {
return a.err
}
// setup app logger:
if err := a.di.Invoke(func(l *zap.Logger) {
a.log = l
}); err != nil {
return err
}
return a.di.Invoke(a.runner)
}
// New is an application constructor.
func New(s *Settings, mod module.Module) App {
var (
a app
err error
)
a.di = dig.New(dig.DeferAcyclicVerification())
a.runner = s.Runner
if s.Prefix == "" {
s.Prefix = s.Name
}
mod = mod.Append(
module.Module{
{Constructor: logger.NewLogger},
{Constructor: NewGracefulContext},
{Constructor: func() (*viper.Viper, error) {
return config.NewConfig(config.Params{
File: s.File,
Type: s.Type,
Prefix: strings.ToUpper(s.Prefix),
Name: s.Name,
Version: fmt.Sprintf("%s(%s)", s.Version, s.Build),
AppDefaults: s.AppDefaults,
})
}},
})
if err = module.Provide(a.di, mod); err != nil {
a.err = err
}
return &a
}

26
lib/fix/grace.go Normal file
View file

@ -0,0 +1,26 @@
package fix
import (
"context"
"os"
"os/signal"
"syscall"
"go.uber.org/zap"
)
// NewGracefulContext returns graceful context.
func NewGracefulContext(l *zap.Logger) context.Context {
ctx, cancel := context.WithCancel(context.Background())
go func() {
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
sig := <-ch
l.Info("received signal",
zap.String("signal", sig.String()))
cancel()
}()
return ctx
}

90
lib/fix/logger/logger.go Normal file
View file

@ -0,0 +1,90 @@
package logger
import (
"strings"
"github.com/spf13/viper"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
const (
formatJSON = "json"
formatConsole = "console"
defaultSamplingInitial = 100
defaultSamplingThereafter = 100
)
func safeLevel(lvl string) zap.AtomicLevel {
switch strings.ToLower(lvl) {
case "debug":
return zap.NewAtomicLevelAt(zap.DebugLevel)
case "warn":
return zap.NewAtomicLevelAt(zap.WarnLevel)
case "error":
return zap.NewAtomicLevelAt(zap.ErrorLevel)
case "fatal":
return zap.NewAtomicLevelAt(zap.FatalLevel)
case "panic":
return zap.NewAtomicLevelAt(zap.PanicLevel)
default:
return zap.NewAtomicLevelAt(zap.InfoLevel)
}
}
// NewLogger is a logger's constructor.
func NewLogger(v *viper.Viper) (*zap.Logger, error) {
c := zap.NewProductionConfig()
c.OutputPaths = []string{"stdout"}
c.ErrorOutputPaths = []string{"stdout"}
if v.IsSet("logger.sampling") {
c.Sampling = &zap.SamplingConfig{
Initial: defaultSamplingInitial,
Thereafter: defaultSamplingThereafter,
}
if val := v.GetInt("logger.sampling.initial"); val > 0 {
c.Sampling.Initial = val
}
if val := v.GetInt("logger.sampling.thereafter"); val > 0 {
c.Sampling.Thereafter = val
}
}
// logger level
c.Level = safeLevel(v.GetString("logger.level"))
traceLvl := safeLevel(v.GetString("logger.trace_level"))
// logger format
switch f := v.GetString("logger.format"); strings.ToLower(f) {
case formatConsole:
c.Encoding = formatConsole
default:
c.Encoding = formatJSON
}
// logger time
c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
l, err := c.Build(
// enable trace only for current log-level
zap.AddStacktrace(traceLvl))
if err != nil {
return nil, err
}
if v.GetBool("logger.no_disclaimer") {
return l, nil
}
name := v.GetString("app.name")
version := v.GetString("app.version")
return l.With(
zap.String("app_name", name),
zap.String("app_version", version)), nil
}

35
lib/fix/module/module.go Normal file
View file

@ -0,0 +1,35 @@
package module
import (
"go.uber.org/dig"
)
type (
// Module type
Module []*Provider
// Provider struct
Provider struct {
Constructor interface{}
Options []dig.ProvideOption
}
)
// Append module to target module and return new module
func (m Module) Append(mods ...Module) Module {
var result = m
for _, mod := range mods {
result = append(result, mod...)
}
return result
}
// Provide set providers functions to DI container
func Provide(dic *dig.Container, providers Module) error {
for _, p := range providers {
if err := dic.Provide(p.Constructor, p.Options...); err != nil {
return err
}
}
return nil
}

46
lib/fix/services.go Normal file
View file

@ -0,0 +1,46 @@
package fix
import (
"context"
)
type (
// Service interface
Service interface {
Start(context.Context)
Stop()
}
combiner []Service
)
var _ Service = (combiner)(nil)
// NewServices creates single runner.
func NewServices(items ...Service) Service {
var svc = make(combiner, 0, len(items))
for _, item := range items {
if item == nil {
continue
}
svc = append(svc, item)
}
return svc
}
// Start all services.
func (c combiner) Start(ctx context.Context) {
for _, svc := range c {
svc.Start(ctx)
}
}
// Stop all services.
func (c combiner) Stop() {
for _, svc := range c {
svc.Stop()
}
}

114
lib/fix/web/http.go Normal file
View file

@ -0,0 +1,114 @@
package web
import (
"context"
"net/http"
"sync/atomic"
"time"
"github.com/spf13/viper"
"go.uber.org/zap"
)
type (
httpParams struct {
Key string
Viper *viper.Viper
Logger *zap.Logger
Handler http.Handler
}
httpServer struct {
name string
started *int32
logger *zap.Logger
shutdownTTL time.Duration
server server
}
)
func (h *httpServer) Start(ctx context.Context) {
if h == nil {
return
}
if !atomic.CompareAndSwapInt32(h.started, 0, 1) {
h.logger.Info("http: already started",
zap.String("server", h.name))
return
}
go func() {
if err := h.server.serve(ctx); err != nil {
if err != http.ErrServerClosed {
h.logger.Error("http: could not start server",
zap.Error(err))
}
}
}()
}
func (h *httpServer) Stop() {
if h == nil {
return
}
if !atomic.CompareAndSwapInt32(h.started, 1, 0) {
h.logger.Info("http: already stopped",
zap.String("server", h.name))
return
}
ctx, cancel := context.WithTimeout(context.Background(), h.shutdownTTL)
defer cancel()
h.logger.Debug("http: try to stop server",
zap.String("server", h.name))
if err := h.server.shutdown(ctx); err != nil {
h.logger.Error("http: could not stop server",
zap.Error(err))
}
}
const defaultShutdownTTL = 30 * time.Second
func newHTTPServer(p httpParams) *httpServer {
var (
address string
shutdown time.Duration
)
if address = p.Viper.GetString(p.Key + ".address"); address == "" {
p.Logger.Info("Empty bind address, skip",
zap.String("server", p.Key))
return nil
}
if p.Handler == nil {
p.Logger.Info("Empty handler, skip",
zap.String("server", p.Key))
return nil
}
p.Logger.Info("Create http.Server",
zap.String("server", p.Key),
zap.String("address", address))
if shutdown = p.Viper.GetDuration(p.Key + ".shutdown_ttl"); shutdown <= 0 {
shutdown = defaultShutdownTTL
}
return &httpServer{
name: p.Key,
started: new(int32),
logger: p.Logger,
shutdownTTL: shutdown,
server: newServer(params{
Address: address,
Name: p.Key,
Config: p.Viper,
Logger: p.Logger,
Handler: p.Handler,
}),
}
}

32
lib/fix/web/metrics.go Normal file
View file

@ -0,0 +1,32 @@
package web
import (
"context"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/viper"
"go.uber.org/zap"
)
// Metrics is an interface of metric tool.
type Metrics interface {
Start(ctx context.Context)
Stop()
}
const metricsKey = "metrics"
// NewMetrics is a metric tool's constructor.
func NewMetrics(l *zap.Logger, v *viper.Viper) Metrics {
if !v.GetBool(metricsKey + ".enabled") {
l.Debug("metrics server disabled")
return nil
}
return newHTTPServer(httpParams{
Key: metricsKey,
Viper: v,
Logger: l,
Handler: promhttp.Handler(),
})
}

44
lib/fix/web/pprof.go Normal file
View file

@ -0,0 +1,44 @@
package web
import (
"context"
"expvar"
"net/http"
"net/http/pprof"
"github.com/spf13/viper"
"go.uber.org/zap"
)
// Profiler is an interface of profiler.
type Profiler interface {
Start(ctx context.Context)
Stop()
}
const profilerKey = "pprof"
// NewProfiler is a profiler's constructor.
func NewProfiler(l *zap.Logger, v *viper.Viper) Profiler {
if !v.GetBool(profilerKey + ".enabled") {
l.Debug("pprof server disabled")
return nil
}
mux := http.NewServeMux()
mux.Handle("/debug/vars", expvar.Handler())
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
return newHTTPServer(httpParams{
Key: profilerKey,
Viper: v,
Logger: l,
Handler: mux,
})
}

62
lib/fix/web/server.go Normal file
View file

@ -0,0 +1,62 @@
package web
import (
"context"
"net/http"
"github.com/spf13/viper"
"go.uber.org/zap"
)
type (
// Server is an interface of server.
server interface {
serve(ctx context.Context) error
shutdown(ctx context.Context) error
}
contextServer struct {
logger *zap.Logger
server *http.Server
}
params struct {
Address string
Name string
Config *viper.Viper
Logger *zap.Logger
Handler http.Handler
}
)
func newServer(p params) server {
return &contextServer{
logger: p.Logger,
server: &http.Server{
Addr: p.Address,
Handler: p.Handler,
ReadTimeout: p.Config.GetDuration(p.Name + ".read_timeout"),
ReadHeaderTimeout: p.Config.GetDuration(p.Name + ".read_header_timeout"),
WriteTimeout: p.Config.GetDuration(p.Name + ".write_timeout"),
IdleTimeout: p.Config.GetDuration(p.Name + ".idle_timeout"),
MaxHeaderBytes: p.Config.GetInt(p.Name + ".max_header_bytes"),
},
}
}
func (cs *contextServer) serve(ctx context.Context) error {
go func() {
<-ctx.Done()
if err := cs.server.Close(); err != nil {
cs.logger.Info("something went wrong",
zap.Error(err))
}
}()
return cs.server.ListenAndServe()
}
func (cs *contextServer) shutdown(ctx context.Context) error {
return cs.server.Shutdown(ctx)
}

79
lib/fix/worker/worker.go Normal file
View file

@ -0,0 +1,79 @@
package worker
import (
"context"
"sync"
"sync/atomic"
"time"
)
type (
// Workers is an interface of worker tool.
Workers interface {
Start(context.Context)
Stop()
Add(Job Handler)
}
workers struct {
cancel context.CancelFunc
started *int32
wg *sync.WaitGroup
jobs []Handler
}
// Handler is a worker's handling function.
Handler func(ctx context.Context)
// Jobs is a map of worker names to handlers.
Jobs map[string]Handler
// Job groups the parameters of worker's job.
Job struct {
Disabled bool
Immediately bool
Timer time.Duration
Ticker time.Duration
Handler Handler
}
)
// New is a constructor of workers.
func New() Workers {
return &workers{
started: new(int32),
wg: new(sync.WaitGroup),
}
}
func (w *workers) Add(job Handler) {
w.jobs = append(w.jobs, job)
}
func (w *workers) Stop() {
if !atomic.CompareAndSwapInt32(w.started, 1, 0) {
// already stopped
return
}
w.cancel()
w.wg.Wait()
}
func (w *workers) Start(ctx context.Context) {
if !atomic.CompareAndSwapInt32(w.started, 0, 1) {
// already started
return
}
ctx, w.cancel = context.WithCancel(ctx)
for _, job := range w.jobs {
w.wg.Add(1)
go func(handler Handler) {
defer w.wg.Done()
handler(ctx)
}(job)
}
}

392
lib/implementations/acl.go Normal file
View file

@ -0,0 +1,392 @@
package implementations
import (
"context"
sc "github.com/nspcc-dev/neo-go/pkg/smartcontract"
libacl "github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/acl"
"github.com/nspcc-dev/neofs-node/lib/blockchain/goclient"
"github.com/nspcc-dev/neofs-node/lib/container"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/pkg/errors"
)
// Consider moving ACLHelper implementation to the ACL library.
type (
// ACLHelper is an interface, that provides useful functions
// for ACL object pre-processor.
ACLHelper interface {
BasicACLGetter
ContainerOwnerChecker
}
// BasicACLGetter helper provides function to return basic ACL value.
BasicACLGetter interface {
GetBasicACL(context.Context, CID) (uint32, error)
}
// ContainerOwnerChecker checks owner of the container.
ContainerOwnerChecker interface {
IsContainerOwner(context.Context, CID, refs.OwnerID) (bool, error)
}
aclHelper struct {
cnr container.Storage
}
)
type binaryEACLSource struct {
binaryStore acl.BinaryExtendedACLSource
}
// StaticContractClient is a wrapper over Neo:Morph client
// that invokes single smart contract methods with fixed fee.
type StaticContractClient struct {
// neo-go client instance
client *goclient.Client
// contract script-hash
scScriptHash util.Uint160
// invocation fee
fee util.Fixed8
}
// MorphContainerContract is a wrapper over StaticContractClient
// for Container contract calls.
type MorphContainerContract struct {
// NeoFS Container smart-contract
containerContract StaticContractClient
// set EACL method name of container contract
eaclSetMethodName string
// get EACL method name of container contract
eaclGetMethodName string
// get container method name of container contract
cnrGetMethodName string
// put container method name of container contract
cnrPutMethodName string
// delete container method name of container contract
cnrDelMethodName string
// list containers method name of container contract
cnrListMethodName string
}
const (
errNewACLHelper = internal.Error("cannot create ACLHelper instance")
)
// GetBasicACL returns basic ACL of the container.
func (h aclHelper) GetBasicACL(ctx context.Context, cid CID) (uint32, error) {
gp := container.GetParams{}
gp.SetContext(ctx)
gp.SetCID(cid)
gResp, err := h.cnr.GetContainer(gp)
if err != nil {
return 0, err
}
return gResp.Container().BasicACL, nil
}
// IsContainerOwner returns true if provided id is an owner container.
func (h aclHelper) IsContainerOwner(ctx context.Context, cid CID, id refs.OwnerID) (bool, error) {
gp := container.GetParams{}
gp.SetContext(ctx)
gp.SetCID(cid)
gResp, err := h.cnr.GetContainer(gp)
if err != nil {
return false, err
}
return gResp.Container().OwnerID.Equal(id), nil
}
// NewACLHelper returns implementation of the ACLHelper interface.
func NewACLHelper(cnr container.Storage) (ACLHelper, error) {
if cnr == nil {
return nil, errNewACLHelper
}
return aclHelper{cnr}, nil
}
// ExtendedACLSourceFromBinary wraps BinaryExtendedACLSource and returns ExtendedACLSource.
//
// If passed BinaryExtendedACLSource is nil, acl.ErrNilBinaryExtendedACLStore returns.
func ExtendedACLSourceFromBinary(v acl.BinaryExtendedACLSource) (acl.ExtendedACLSource, error) {
if v == nil {
return nil, acl.ErrNilBinaryExtendedACLStore
}
return &binaryEACLSource{
binaryStore: v,
}, nil
}
// GetExtendedACLTable receives eACL table in a binary representation from storage,
// unmarshals it and returns ExtendedACLTable interface.
func (s binaryEACLSource) GetExtendedACLTable(ctx context.Context, cid refs.CID) (libacl.ExtendedACLTable, error) {
key := acl.BinaryEACLKey{}
key.SetCID(cid)
val, err := s.binaryStore.GetBinaryEACL(ctx, key)
if err != nil {
return nil, err
}
eacl := val.EACL()
// TODO: verify signature
res := libacl.WrapEACLTable(nil)
return res, res.UnmarshalBinary(eacl)
}
// NewStaticContractClient initializes a new StaticContractClient.
//
// If passed Client is nil, goclient.ErrNilClient returns.
func NewStaticContractClient(client *goclient.Client, scHash util.Uint160, fee util.Fixed8) (StaticContractClient, error) {
res := StaticContractClient{
client: client,
scScriptHash: scHash,
fee: fee,
}
var err error
if client == nil {
err = goclient.ErrNilClient
}
return res, err
}
// Invoke calls Invoke method of goclient with predefined script hash and fee.
// Supported args types are the same as in goclient.
//
// If Client is not initialized, goclient.ErrNilClient returns.
func (s StaticContractClient) Invoke(method string, args ...interface{}) error {
if s.client == nil {
return goclient.ErrNilClient
}
return s.client.Invoke(
s.scScriptHash,
s.fee,
method,
args...,
)
}
// TestInvoke calls TestInvoke method of goclient with predefined script hash.
//
// If Client is not initialized, goclient.ErrNilClient returns.
func (s StaticContractClient) TestInvoke(method string, args ...interface{}) ([]sc.Parameter, error) {
if s.client == nil {
return nil, goclient.ErrNilClient
}
return s.client.TestInvoke(
s.scScriptHash,
method,
args...,
)
}
// SetContainerContractClient is a container contract client setter.
func (s *MorphContainerContract) SetContainerContractClient(v StaticContractClient) {
s.containerContract = v
}
// SetEACLGetMethodName is a container contract Get EACL method name setter.
func (s *MorphContainerContract) SetEACLGetMethodName(v string) {
s.eaclGetMethodName = v
}
// SetEACLSetMethodName is a container contract Set EACL method name setter.
func (s *MorphContainerContract) SetEACLSetMethodName(v string) {
s.eaclSetMethodName = v
}
// SetContainerGetMethodName is a container contract Get method name setter.
func (s *MorphContainerContract) SetContainerGetMethodName(v string) {
s.cnrGetMethodName = v
}
// SetContainerPutMethodName is a container contract Put method name setter.
func (s *MorphContainerContract) SetContainerPutMethodName(v string) {
s.cnrPutMethodName = v
}
// SetContainerDeleteMethodName is a container contract Delete method name setter.
func (s *MorphContainerContract) SetContainerDeleteMethodName(v string) {
s.cnrDelMethodName = v
}
// SetContainerListMethodName is a container contract List method name setter.
func (s *MorphContainerContract) SetContainerListMethodName(v string) {
s.cnrListMethodName = v
}
// GetBinaryEACL performs the test invocation call of GetEACL method of NeoFS Container contract.
func (s *MorphContainerContract) GetBinaryEACL(_ context.Context, key acl.BinaryEACLKey) (acl.BinaryEACLValue, error) {
res := acl.BinaryEACLValue{}
prms, err := s.containerContract.TestInvoke(
s.eaclGetMethodName,
key.CID().Bytes(),
)
if err != nil {
return res, err
} else if ln := len(prms); ln != 1 {
return res, errors.Errorf("unexpected stack parameter count: %d", ln)
}
eacl, err := goclient.BytesFromStackParameter(prms[0])
if err == nil {
res.SetEACL(eacl)
}
return res, err
}
// PutBinaryEACL invokes the call of SetEACL method of NeoFS Container contract.
func (s *MorphContainerContract) PutBinaryEACL(_ context.Context, key acl.BinaryEACLKey, val acl.BinaryEACLValue) error {
return s.containerContract.Invoke(
s.eaclSetMethodName,
key.CID().Bytes(),
val.EACL(),
val.Signature(),
)
}
// GetContainer performs the test invocation call of Get method of NeoFS Container contract.
func (s *MorphContainerContract) GetContainer(p container.GetParams) (*container.GetResult, error) {
prms, err := s.containerContract.TestInvoke(
s.cnrGetMethodName,
p.CID().Bytes(),
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count: %d", ln)
}
cnrBytes, err := goclient.BytesFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get byte array from stack item")
}
cnr := new(container.Container)
if err := cnr.Unmarshal(cnrBytes); err != nil {
return nil, errors.Wrap(err, "could not unmarshal container from bytes")
}
res := new(container.GetResult)
res.SetContainer(cnr)
return res, nil
}
// PutContainer invokes the call of Put method of NeoFS Container contract.
func (s *MorphContainerContract) PutContainer(p container.PutParams) (*container.PutResult, error) {
cnr := p.Container()
cid, err := cnr.ID()
if err != nil {
return nil, errors.Wrap(err, "could not calculate container ID")
}
cnrBytes, err := cnr.Marshal()
if err != nil {
return nil, errors.Wrap(err, "could not marshal container")
}
if err := s.containerContract.Invoke(
s.cnrPutMethodName,
cnr.OwnerID.Bytes(),
cnrBytes,
[]byte{},
); err != nil {
return nil, errors.Wrap(err, "could not invoke contract method")
}
res := new(container.PutResult)
res.SetCID(cid)
return res, nil
}
// DeleteContainer invokes the call of Delete method of NeoFS Container contract.
func (s *MorphContainerContract) DeleteContainer(p container.DeleteParams) (*container.DeleteResult, error) {
if err := s.containerContract.Invoke(
s.cnrDelMethodName,
p.CID().Bytes(),
p.OwnerID().Bytes(),
[]byte{},
); err != nil {
return nil, errors.Wrap(err, "could not invoke contract method")
}
return new(container.DeleteResult), nil
}
// ListContainers performs the test invocation call of Get method of NeoFS Container contract.
//
// If owner ID list in parameters is non-empty, bytes of first owner are attached to call.
func (s *MorphContainerContract) ListContainers(p container.ListParams) (*container.ListResult, error) {
args := make([]interface{}, 0, 1)
if ownerIDList := p.OwnerIDList(); len(ownerIDList) > 0 {
args = append(args, ownerIDList[0].Bytes())
}
prms, err := s.containerContract.TestInvoke(
s.cnrListMethodName,
args...,
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count: %d", ln)
}
prms, err = goclient.ArrayFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get stack item array from stack item")
}
cidList := make([]CID, 0, len(prms))
for i := range prms {
cidBytes, err := goclient.BytesFromStackParameter(prms[i])
if err != nil {
return nil, errors.Wrap(err, "could not get byte array from stack item")
}
cid, err := refs.CIDFromBytes(cidBytes)
if err != nil {
return nil, errors.Wrap(err, "could not get container ID from bytes")
}
cidList = append(cidList, cid)
}
res := new(container.ListResult)
res.SetCIDList(cidList)
return res, nil
}

View file

@ -0,0 +1,19 @@
package implementations
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestStaticContractClient(t *testing.T) {
s := new(StaticContractClient)
require.NotPanics(t, func() {
_, _ = s.TestInvoke("")
})
require.NotPanics(t, func() {
_ = s.Invoke("")
})
}

View file

@ -0,0 +1,141 @@
package implementations
import (
"github.com/nspcc-dev/neo-go/pkg/encoding/address"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-node/lib/blockchain/goclient"
"github.com/pkg/errors"
)
// MorphBalanceContract is a wrapper over NeoFS Balance contract client
// that provides an interface of manipulations with user funds.
type MorphBalanceContract struct {
// NeoFS Balance smart-contract
balanceContract StaticContractClient
// "balance of" method name of balance contract
balanceOfMethodName string
// decimals method name of balance contract
decimalsMethodName string
}
// BalanceOfParams is a structure that groups the parameters
// for NeoFS user balance receiving operation.
type BalanceOfParams struct {
owner refs.OwnerID
}
// BalanceOfResult is a structure that groups the values
// of the result of NeoFS user balance receiving operation.
type BalanceOfResult struct {
amount int64
}
// DecimalsParams is a structure that groups the parameters
// for NeoFS token decimals receiving operation.
type DecimalsParams struct {
}
// DecimalsResult is a structure that groups the values
// of the result of NeoFS token decimals receiving operation.
type DecimalsResult struct {
dec int64
}
// SetBalanceContractClient is a Balance contract client setter.
func (s *MorphBalanceContract) SetBalanceContractClient(v StaticContractClient) {
s.balanceContract = v
}
// SetBalanceOfMethodName is a Balance contract balanceOf method name setter.
func (s *MorphBalanceContract) SetBalanceOfMethodName(v string) {
s.balanceOfMethodName = v
}
// SetDecimalsMethodName is a Balance contract decimals method name setter.
func (s *MorphBalanceContract) SetDecimalsMethodName(v string) {
s.decimalsMethodName = v
}
// BalanceOf performs the test invocation call of balanceOf method of NeoFS Balance contract.
func (s MorphBalanceContract) BalanceOf(p BalanceOfParams) (*BalanceOfResult, error) {
owner := p.OwnerID()
u160, err := address.StringToUint160(owner.String())
if err != nil {
return nil, errors.Wrap(err, "could not convert wallet address to Uint160")
}
prms, err := s.balanceContract.TestInvoke(
s.balanceOfMethodName,
u160.BytesBE(),
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count (balanceOf): %d", ln)
}
amount, err := goclient.IntFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get integer stack item from stack item (amount)")
}
res := new(BalanceOfResult)
res.SetAmount(amount)
return res, nil
}
// Decimals performs the test invocation call of decimals method of NeoFS Balance contract.
func (s MorphBalanceContract) Decimals(DecimalsParams) (*DecimalsResult, error) {
prms, err := s.balanceContract.TestInvoke(
s.decimalsMethodName,
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count (decimals): %d", ln)
}
dec, err := goclient.IntFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get integer stack item from stack item (decimal)")
}
res := new(DecimalsResult)
res.SetDecimals(dec)
return res, nil
}
// SetOwnerID is an owner ID setter.
func (s *BalanceOfParams) SetOwnerID(v refs.OwnerID) {
s.owner = v
}
// OwnerID is an owner ID getter.
func (s BalanceOfParams) OwnerID() refs.OwnerID {
return s.owner
}
// SetAmount is an funds amount setter.
func (s *BalanceOfResult) SetAmount(v int64) {
s.amount = v
}
// Amount is an funds amount getter.
func (s BalanceOfResult) Amount() int64 {
return s.amount
}
// SetDecimals is a decimals setter.
func (s *DecimalsResult) SetDecimals(v int64) {
s.dec = v
}
// Decimals is a decimals getter.
func (s DecimalsResult) Decimals() int64 {
return s.dec
}

View file

@ -0,0 +1,35 @@
package implementations
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/stretchr/testify/require"
)
func TestBalanceOfParams(t *testing.T) {
s := BalanceOfParams{}
owner := refs.OwnerID{1, 2, 3}
s.SetOwnerID(owner)
require.Equal(t, owner, s.OwnerID())
}
func TestBalanceOfResult(t *testing.T) {
s := BalanceOfResult{}
amount := int64(100)
s.SetAmount(amount)
require.Equal(t, amount, s.Amount())
}
func TestDecimalsResult(t *testing.T) {
s := DecimalsResult{}
dec := int64(100)
s.SetDecimals(dec)
require.Equal(t, dec, s.Decimals())
}

View file

@ -0,0 +1,311 @@
package implementations
import (
"github.com/nspcc-dev/neo-go/pkg/smartcontract"
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/nspcc-dev/neofs-node/lib/blockchain/goclient"
"github.com/nspcc-dev/neofs-node/lib/boot"
"github.com/nspcc-dev/neofs-node/lib/ir"
"github.com/nspcc-dev/neofs-node/lib/netmap"
"github.com/pkg/errors"
)
// MorphNetmapContract is a wrapper over NeoFS Netmap contract client
// that provides an interface of network map manipulations.
type MorphNetmapContract struct {
// NeoFS Netmap smart-contract
netmapContract StaticContractClient
// add peer method name of netmap contract
addPeerMethodName string
// new epoch method name of netmap contract
newEpochMethodName string
// get netmap method name of netmap contract
getNetMapMethodName string
// update state method name of netmap contract
updStateMethodName string
// IR list method name of netmap contract
irListMethodName string
}
// UpdateEpochParams is a structure that groups the parameters
// for NeoFS epoch number updating.
type UpdateEpochParams struct {
epoch uint64
}
// UpdateStateParams is a structure that groups the parameters
// for NeoFS node state updating.
type UpdateStateParams struct {
st NodeState
key []byte
}
// NodeState is a type of node states enumeration.
type NodeState int64
const (
_ NodeState = iota
// StateOffline is an offline node state value.
StateOffline
)
const addPeerFixedArgNumber = 2
const nodeInfoFixedPrmNumber = 3
// SetNetmapContractClient is a Netmap contract client setter.
func (s *MorphNetmapContract) SetNetmapContractClient(v StaticContractClient) {
s.netmapContract = v
}
// SetAddPeerMethodName is a Netmap contract AddPeer method name setter.
func (s *MorphNetmapContract) SetAddPeerMethodName(v string) {
s.addPeerMethodName = v
}
// SetNewEpochMethodName is a Netmap contract NewEpoch method name setter.
func (s *MorphNetmapContract) SetNewEpochMethodName(v string) {
s.newEpochMethodName = v
}
// SetNetMapMethodName is a Netmap contract Netmap method name setter.
func (s *MorphNetmapContract) SetNetMapMethodName(v string) {
s.getNetMapMethodName = v
}
// SetUpdateStateMethodName is a Netmap contract UpdateState method name setter.
func (s *MorphNetmapContract) SetUpdateStateMethodName(v string) {
s.updStateMethodName = v
}
// SetIRListMethodName is a Netmap contract InnerRingList method name setter.
func (s *MorphNetmapContract) SetIRListMethodName(v string) {
s.irListMethodName = v
}
// AddPeer invokes the call of AddPeer method of NeoFS Netmap contract.
func (s *MorphNetmapContract) AddPeer(p boot.BootstrapPeerParams) error {
info := p.NodeInfo()
opts := info.GetOptions()
args := make([]interface{}, 0, addPeerFixedArgNumber+len(opts))
args = append(args,
// Address
[]byte(info.GetAddress()),
// Public key
info.GetPubKey(),
)
// Options
for i := range opts {
args = append(args, []byte(opts[i]))
}
return s.netmapContract.Invoke(
s.addPeerMethodName,
args...,
)
}
// UpdateEpoch invokes the call of NewEpoch method of NeoFS Netmap contract.
func (s *MorphNetmapContract) UpdateEpoch(p UpdateEpochParams) error {
return s.netmapContract.Invoke(
s.newEpochMethodName,
int64(p.Number()), // TODO: do not cast after uint64 type will become supported in client
)
}
// GetNetMap performs the test invocation call of Netmap method of NeoFS Netmap contract.
func (s *MorphNetmapContract) GetNetMap(p netmap.GetParams) (*netmap.GetResult, error) {
prms, err := s.netmapContract.TestInvoke(
s.getNetMapMethodName,
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count (Nodes): %d", ln)
}
prms, err = goclient.ArrayFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get stack item array from stack item (Nodes)")
}
nm := netmap.NewNetmap()
for i := range prms {
nodeInfo, err := nodeInfoFromStackItem(prms[i])
if err != nil {
return nil, errors.Wrapf(err, "could not parse stack item (Node #%d)", i)
}
if err := nm.AddNode(nodeInfo); err != nil {
return nil, errors.Wrapf(err, "could not add node #%d to network map", i)
}
}
res := new(netmap.GetResult)
res.SetNetMap(nm)
return res, nil
}
func nodeInfoFromStackItem(prm smartcontract.Parameter) (*bootstrap.NodeInfo, error) {
prms, err := goclient.ArrayFromStackParameter(prm)
if err != nil {
return nil, errors.Wrapf(err, "could not get stack item array (NodeInfo)")
} else if ln := len(prms); ln != nodeInfoFixedPrmNumber {
return nil, errors.Errorf("unexpected stack item count (NodeInfo): expected %d, has %d", 3, ln)
}
res := new(bootstrap.NodeInfo)
// Address
addrBytes, err := goclient.BytesFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get byte array from stack item (Address)")
}
res.Address = string(addrBytes)
// Public key
res.PubKey, err = goclient.BytesFromStackParameter(prms[1])
if err != nil {
return nil, errors.Wrap(err, "could not get byte array from stack item (Public key)")
}
// Options
prms, err = goclient.ArrayFromStackParameter(prms[2])
if err != nil {
return nil, errors.Wrapf(err, "could not get stack item array (Options)")
}
res.Options = make([]string, 0, len(prms))
for i := range prms {
optBytes, err := goclient.BytesFromStackParameter(prms[i])
if err != nil {
return nil, errors.Wrapf(err, "could not get byte array from stack item (Option #%d)", i)
}
res.Options = append(res.Options, string(optBytes))
}
return res, nil
}
// UpdateState invokes the call of UpdateState method of NeoFS Netmap contract.
func (s *MorphNetmapContract) UpdateState(p UpdateStateParams) error {
return s.netmapContract.Invoke(
s.updStateMethodName,
p.State().Int64(),
p.Key(),
)
}
// GetIRInfo performs the test invocation call of InnerRingList method of NeoFS Netmap contract.
func (s *MorphNetmapContract) GetIRInfo(ir.GetInfoParams) (*ir.GetInfoResult, error) {
prms, err := s.netmapContract.TestInvoke(
s.irListMethodName,
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count (Nodes): %d", ln)
}
irInfo, err := irInfoFromStackItem(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get IR info from stack item")
}
res := new(ir.GetInfoResult)
res.SetInfo(*irInfo)
return res, nil
}
func irInfoFromStackItem(prm smartcontract.Parameter) (*ir.Info, error) {
prms, err := goclient.ArrayFromStackParameter(prm)
if err != nil {
return nil, errors.Wrap(err, "could not get stack item array")
}
nodes := make([]ir.Node, 0, len(prms))
for i := range prms {
node, err := irNodeFromStackItem(prms[i])
if err != nil {
return nil, errors.Wrapf(err, "could not get node info from stack item (IRNode #%d)", i)
}
nodes = append(nodes, *node)
}
info := new(ir.Info)
info.SetNodes(nodes)
return info, nil
}
func irNodeFromStackItem(prm smartcontract.Parameter) (*ir.Node, error) {
prms, err := goclient.ArrayFromStackParameter(prm)
if err != nil {
return nil, errors.Wrap(err, "could not get stack item array (IRNode)")
}
// Public key
keyBytes, err := goclient.BytesFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get byte array from stack item (Key)")
}
node := new(ir.Node)
node.SetKey(keyBytes)
return node, nil
}
// SetNumber is an epoch number setter.
func (s *UpdateEpochParams) SetNumber(v uint64) {
s.epoch = v
}
// Number is an epoch number getter.
func (s UpdateEpochParams) Number() uint64 {
return s.epoch
}
// SetState is a state setter.
func (s *UpdateStateParams) SetState(v NodeState) {
s.st = v
}
// State is a state getter.
func (s UpdateStateParams) State() NodeState {
return s.st
}
// SetKey is a public key setter.
func (s *UpdateStateParams) SetKey(v []byte) {
s.key = v
}
// Key is a public key getter.
func (s UpdateStateParams) Key() []byte {
return s.key
}
// Int64 converts NodeState to int64.
func (s NodeState) Int64() int64 {
return int64(s)
}

View file

@ -0,0 +1,30 @@
package implementations
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestUpdateEpochParams(t *testing.T) {
s := UpdateEpochParams{}
e := uint64(100)
s.SetNumber(e)
require.Equal(t, e, s.Number())
}
func TestUpdateStateParams(t *testing.T) {
s := UpdateStateParams{}
st := NodeState(1)
s.SetState(st)
require.Equal(t, st, s.State())
key := []byte{1, 2, 3}
s.SetKey(key)
require.Equal(t, key, s.Key())
}

View file

@ -0,0 +1,7 @@
package implementations
// EpochReceiver is an interface of the container
// of NeoFS epoch number with read access.
type EpochReceiver interface {
Epoch() uint64
}

View file

@ -0,0 +1,78 @@
package implementations
import (
"context"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-api-go/query"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-api-go/service"
"github.com/nspcc-dev/neofs-node/lib/replication"
"github.com/nspcc-dev/neofs-node/lib/transport"
"github.com/pkg/errors"
"go.uber.org/zap"
)
type (
locator struct {
executor SelectiveContainerExecutor
log *zap.Logger
}
// LocatorParams groups the parameters of ObjectLocator constructor.
LocatorParams struct {
SelectiveContainerExecutor SelectiveContainerExecutor
Logger *zap.Logger
}
)
const locatorInstanceFailMsg = "could not create object locator"
var errEmptyObjectsContainerHandler = errors.New("empty container objects container handler")
func (s *locator) LocateObject(ctx context.Context, addr Address) (res []multiaddr.Multiaddr, err error) {
queryBytes, err := (&query.Query{
Filters: []query.Filter{
{
Type: query.Filter_Exact,
Name: transport.KeyID,
Value: addr.ObjectID.String(),
},
},
}).Marshal()
if err != nil {
return nil, errors.Wrap(err, "locate object failed on query marshal")
}
err = s.executor.Search(ctx, &SearchParams{
SelectiveParams: SelectiveParams{
CID: addr.CID,
TTL: service.NonForwardingTTL,
IDList: make([]ObjectID, 1),
},
SearchCID: addr.CID,
SearchQuery: queryBytes,
Handler: func(node multiaddr.Multiaddr, addrList []refs.Address) {
if len(addrList) > 0 {
res = append(res, node)
}
},
})
return
}
// NewObjectLocator constructs replication.ObjectLocator from SelectiveContainerExecutor.
func NewObjectLocator(p LocatorParams) (replication.ObjectLocator, error) {
switch {
case p.SelectiveContainerExecutor == nil:
return nil, errors.Wrap(errEmptyObjectsContainerHandler, locatorInstanceFailMsg)
case p.Logger == nil:
return nil, errors.Wrap(errEmptyLogger, locatorInstanceFailMsg)
}
return &locator{
executor: p.SelectiveContainerExecutor,
log: p.Logger,
}, nil
}

View file

@ -0,0 +1,38 @@
package implementations
import (
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
type testExecutor struct {
SelectiveContainerExecutor
}
func TestNewObjectLocator(t *testing.T) {
validParams := LocatorParams{
SelectiveContainerExecutor: new(testExecutor),
Logger: zap.L(),
}
t.Run("valid params", func(t *testing.T) {
s, err := NewObjectLocator(validParams)
require.NoError(t, err)
require.NotNil(t, s)
})
t.Run("empty logger", func(t *testing.T) {
p := validParams
p.Logger = nil
_, err := NewObjectLocator(p)
require.EqualError(t, err, errors.Wrap(errEmptyLogger, locatorInstanceFailMsg).Error())
})
t.Run("empty container handler", func(t *testing.T) {
p := validParams
p.SelectiveContainerExecutor = nil
_, err := NewObjectLocator(p)
require.EqualError(t, err, errors.Wrap(errEmptyObjectsContainerHandler, locatorInstanceFailMsg).Error())
})
}

View file

@ -0,0 +1,131 @@
package implementations
import (
"context"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-api-go/service"
"github.com/nspcc-dev/neofs-node/lib/localstore"
"github.com/nspcc-dev/neofs-node/lib/replication"
"github.com/pkg/errors"
"go.uber.org/zap"
)
type (
// ObjectStorage is an interface of encapsulated ObjectReceptacle and ObjectSource pair.
ObjectStorage interface {
replication.ObjectReceptacle
replication.ObjectSource
}
objectStorage struct {
ls localstore.Localstore
executor SelectiveContainerExecutor
log *zap.Logger
}
// ObjectStorageParams groups the parameters of ObjectStorage constructor.
ObjectStorageParams struct {
Localstore localstore.Localstore
SelectiveContainerExecutor SelectiveContainerExecutor
Logger *zap.Logger
}
)
const objectSourceInstanceFailMsg = "could not create object source"
var errNilObject = errors.New("object is nil")
var errCouldNotGetObject = errors.New("could not get object from any node")
func (s *objectStorage) Put(ctx context.Context, params replication.ObjectStoreParams) error {
if params.Object == nil {
return errNilObject
} else if len(params.Nodes) == 0 {
if s.ls == nil {
return errEmptyLocalstore
}
return s.ls.Put(ctx, params.Object)
}
nodes := make([]multiaddr.Multiaddr, len(params.Nodes))
for i := range params.Nodes {
nodes[i] = params.Nodes[i].Node
}
return s.executor.Put(ctx, &PutParams{
SelectiveParams: SelectiveParams{
CID: params.Object.SystemHeader.CID,
Nodes: nodes,
TTL: service.NonForwardingTTL,
IDList: make([]ObjectID, 1),
},
Object: params.Object,
Handler: func(node multiaddr.Multiaddr, valid bool) {
if params.Handler == nil {
return
}
for i := range params.Nodes {
if params.Nodes[i].Node.Equal(node) {
params.Handler(params.Nodes[i], valid)
return
}
}
},
})
}
func (s *objectStorage) Get(ctx context.Context, addr Address) (res *Object, err error) {
if s.ls != nil {
if has, err := s.ls.Has(addr); err == nil && has {
if res, err = s.ls.Get(addr); err == nil {
return res, err
}
}
}
if err = s.executor.Get(ctx, &GetParams{
SelectiveParams: SelectiveParams{
CID: addr.CID,
TTL: service.NonForwardingTTL,
IDList: []ObjectID{addr.ObjectID},
Breaker: func(refs.Address) (cFlag ProgressControlFlag) {
if res != nil {
cFlag = BreakProgress
}
return
},
},
Handler: func(node multiaddr.Multiaddr, obj *object.Object) { res = obj },
}); err != nil {
return
} else if res == nil {
return nil, errCouldNotGetObject
}
return
}
// NewObjectStorage encapsulates Localstore and SelectiveContainerExecutor
// and returns ObjectStorage interface.
func NewObjectStorage(p ObjectStorageParams) (ObjectStorage, error) {
if p.Logger == nil {
return nil, errors.Wrap(errEmptyLogger, objectSourceInstanceFailMsg)
}
if p.Localstore == nil {
p.Logger.Warn("local storage not provided")
}
if p.SelectiveContainerExecutor == nil {
p.Logger.Warn("object container handler not provided")
}
return &objectStorage{
ls: p.Localstore,
executor: p.SelectiveContainerExecutor,
log: p.Logger,
}, nil
}

View file

@ -0,0 +1,74 @@
package implementations
import (
"crypto/ecdsa"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/peers"
"github.com/pkg/errors"
"go.uber.org/zap"
)
type (
// AddressStoreComponent is an interface of encapsulated AddressStore and NodePublicKeyReceiver pair.
AddressStoreComponent interface {
AddressStore
NodePublicKeyReceiver
}
// AddressStore is an interface of the container of local Multiaddr.
AddressStore interface {
SelfAddr() (multiaddr.Multiaddr, error)
}
// NodePublicKeyReceiver is an interface of Multiaddr to PublicKey converter.
NodePublicKeyReceiver interface {
PublicKey(multiaddr.Multiaddr) *ecdsa.PublicKey
}
addressStore struct {
ps peers.Store
log *zap.Logger
}
)
const (
addressStoreInstanceFailMsg = "could not create address store"
errEmptyPeerStore = internal.Error("empty peer store")
errEmptyAddressStore = internal.Error("empty address store")
)
func (s addressStore) SelfAddr() (multiaddr.Multiaddr, error) { return s.ps.GetAddr(s.ps.SelfID()) }
func (s addressStore) PublicKey(mAddr multiaddr.Multiaddr) (res *ecdsa.PublicKey) {
if peerID, err := s.ps.AddressID(mAddr); err != nil {
s.log.Error("could not peer ID",
zap.Stringer("node", mAddr),
zap.Error(err),
)
} else if res, err = s.ps.GetPublicKey(peerID); err != nil {
s.log.Error("could not receive public key",
zap.Stringer("peer", peerID),
zap.Error(err),
)
}
return res
}
// NewAddressStore wraps peer store and returns AddressStoreComponent.
func NewAddressStore(ps peers.Store, log *zap.Logger) (AddressStoreComponent, error) {
if ps == nil {
return nil, errors.Wrap(errEmptyPeerStore, addressStoreInstanceFailMsg)
} else if log == nil {
return nil, errors.Wrap(errEmptyLogger, addressStoreInstanceFailMsg)
}
return &addressStore{
ps: ps,
log: log,
}, nil
}

View file

@ -0,0 +1,152 @@
package implementations
import (
"context"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/nspcc-dev/neofs-api-go/container"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/netmap"
"github.com/nspcc-dev/neofs-node/lib/placement"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
/*
File source code includes implementations of placement-related solutions.
Highly specialized interfaces give the opportunity to hide placement implementation in a black box for the reasons:
* placement is implementation-tied entity working with graphs, filters, etc.;
* NeoFS components are mostly needed in a small part of the solutions provided by placement;
* direct dependency from placement avoidance helps other components do not touch crucial changes in placement.
*/
type (
// CID is a type alias of
// CID from refs package of neofs-api-go.
CID = refs.CID
// SGID is a type alias of
// SGID from refs package of neofs-api-go.
SGID = refs.SGID
// ObjectID is a type alias of
// ObjectID from refs package of neofs-api-go.
ObjectID = refs.ObjectID
// Object is a type alias of
// Object from object package of neofs-api-go.
Object = object.Object
// Address is a type alias of
// Address from refs package of neofs-api-go.
Address = refs.Address
// Netmap is a type alias of
// NetMap from netmap package.
Netmap = netmap.NetMap
// ObjectPlacer is an interface of placement utility.
ObjectPlacer interface {
ContainerNodesLister
ContainerInvolvementChecker
GetNodes(ctx context.Context, addr Address, usePreviousNetMap bool, excl ...multiaddr.Multiaddr) ([]multiaddr.Multiaddr, error)
Epoch() uint64
}
// ContainerNodesLister is an interface of container placement vector builder.
ContainerNodesLister interface {
ContainerNodes(ctx context.Context, cid CID) ([]multiaddr.Multiaddr, error)
ContainerNodesInfo(ctx context.Context, cid CID, prev int) ([]bootstrap.NodeInfo, error)
}
// ContainerInvolvementChecker is an interface of container affiliation checker.
ContainerInvolvementChecker interface {
IsContainerNode(ctx context.Context, addr multiaddr.Multiaddr, cid CID, previousNetMap bool) (bool, error)
}
objectPlacer struct {
pl placement.Component
}
)
const errEmptyPlacement = internal.Error("could not create storage lister: empty placement component")
// NewObjectPlacer wraps placement.Component and returns ObjectPlacer interface.
func NewObjectPlacer(pl placement.Component) (ObjectPlacer, error) {
if pl == nil {
return nil, errEmptyPlacement
}
return &objectPlacer{pl}, nil
}
func (v objectPlacer) ContainerNodes(ctx context.Context, cid CID) ([]multiaddr.Multiaddr, error) {
graph, err := v.pl.Query(ctx, placement.ContainerID(cid))
if err != nil {
return nil, errors.Wrap(err, "objectPlacer.ContainerNodes failed on graph query")
}
return graph.NodeList()
}
func (v objectPlacer) ContainerNodesInfo(ctx context.Context, cid CID, prev int) ([]bootstrap.NodeInfo, error) {
graph, err := v.pl.Query(ctx, placement.ContainerID(cid), placement.UsePreviousNetmap(prev))
if err != nil {
return nil, errors.Wrap(err, "objectPlacer.ContainerNodesInfo failed on graph query")
}
return graph.NodeInfo()
}
func (v objectPlacer) GetNodes(ctx context.Context, addr Address, usePreviousNetMap bool, excl ...multiaddr.Multiaddr) ([]multiaddr.Multiaddr, error) {
queryOptions := make([]placement.QueryOption, 1, 2)
queryOptions[0] = placement.ContainerID(addr.CID)
if usePreviousNetMap {
queryOptions = append(queryOptions, placement.UsePreviousNetmap(1))
}
graph, err := v.pl.Query(ctx, queryOptions...)
if err != nil {
if st, ok := status.FromError(errors.Cause(err)); ok && st.Code() == codes.NotFound {
return nil, container.ErrNotFound
}
return nil, errors.Wrap(err, "placer.GetNodes failed on graph query")
}
filter := func(group netmap.SFGroup, bucket *netmap.Bucket) *netmap.Bucket {
return bucket
}
if !addr.ObjectID.Empty() {
filter = func(group netmap.SFGroup, bucket *netmap.Bucket) *netmap.Bucket {
return bucket.GetSelection(group.Selectors, addr.ObjectID.Bytes())
}
}
return graph.Exclude(excl).Filter(filter).NodeList()
}
func (v objectPlacer) IsContainerNode(ctx context.Context, addr multiaddr.Multiaddr, cid CID, previousNetMap bool) (bool, error) {
nodes, err := v.GetNodes(ctx, Address{
CID: cid,
}, previousNetMap)
if err != nil {
return false, errors.Wrap(err, "placer.FromContainer failed on placer.GetNodes")
}
for i := range nodes {
if nodes[i].Equal(addr) {
return true, nil
}
}
return false, nil
}
func (v objectPlacer) Epoch() uint64 { return v.pl.NetworkState().Epoch }

View file

@ -0,0 +1,41 @@
package implementations
import (
"github.com/nspcc-dev/neofs-node/lib/peers"
)
// MorphReputationContract is a wrapper over NeoFS Reputation contract client
// that provides an interface of the storage of global trust values.
type MorphReputationContract struct {
// NeoFS Reputation smart-contract
repContract StaticContractClient
// put method name of reputation contract
putMethodName string
// list method name of reputation contract
listMethodName string
// public key storage
pkStore peers.PublicKeyStore
}
// SetReputationContractClient is a Reputation contract client setter.
func (s *MorphReputationContract) SetReputationContractClient(v StaticContractClient) {
s.repContract = v
}
// SetPublicKeyStore is a public key store setter.
func (s *MorphReputationContract) SetPublicKeyStore(v peers.PublicKeyStore) {
s.pkStore = v
}
// SetPutMethodName is a Reputation contract Put method name setter.
func (s *MorphReputationContract) SetPutMethodName(v string) {
s.putMethodName = v
}
// SetListMethodName is a Reputation contract List method name setter.
func (s *MorphReputationContract) SetListMethodName(v string) {
s.listMethodName = v
}

136
lib/implementations/sg.go Normal file
View file

@ -0,0 +1,136 @@
package implementations
import (
"context"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-api-go/hash"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-api-go/service"
"github.com/nspcc-dev/neofs-api-go/storagegroup"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/pkg/errors"
"go.uber.org/zap"
)
type (
// StorageGroupInfoReceiverParams groups the parameters of
// storage group information receiver.
StorageGroupInfoReceiverParams struct {
SelectiveContainerExecutor SelectiveContainerExecutor
Logger *zap.Logger
}
sgInfoRecv struct {
executor SelectiveContainerExecutor
log *zap.Logger
}
)
const locationFinderInstanceFailMsg = "could not create object location finder"
// ErrIncompleteSGInfo is returned by storage group information receiver
// that could not receive full information.
const ErrIncompleteSGInfo = internal.Error("could not receive full storage group info")
// PublicSessionToken is a context key for SessionToken.
// FIXME: temp solution for cycle import fix.
// Unify with same const from transformer pkg.
const PublicSessionToken = "public token"
// BearerToken is a context key for BearerToken.
const BearerToken = "bearer token"
// ExtendedHeaders is a context key for X-headers.
const ExtendedHeaders = "extended headers"
func (s *sgInfoRecv) GetSGInfo(ctx context.Context, cid CID, group []ObjectID) (*storagegroup.StorageGroup, error) {
var (
err error
res = new(storagegroup.StorageGroup)
hashList = make([]hash.Hash, 0, len(group))
)
m := make(map[string]struct{}, len(group))
for i := range group {
m[group[i].String()] = struct{}{}
}
// FIXME: hardcoded for simplicity.
// Function is called in next cases:
// - SG transformation on trusted node side (only in this case session token is needed);
// - SG info check on container nodes (token is not needed since system group has extra access);
// - data audit on inner ring nodes (same as previous).
var token service.SessionToken
if v, ok := ctx.Value(PublicSessionToken).(service.SessionToken); ok {
token = v
}
var bearer service.BearerToken
if v, ok := ctx.Value(BearerToken).(service.BearerToken); ok {
bearer = v
}
var extHdrs []service.ExtendedHeader
if v, ok := ctx.Value(ExtendedHeaders).([]service.ExtendedHeader); ok {
extHdrs = v
}
if err = s.executor.Head(ctx, &HeadParams{
GetParams: GetParams{
SelectiveParams: SelectiveParams{
CID: cid,
TTL: service.SingleForwardingTTL,
IDList: group,
Breaker: func(addr refs.Address) (cFlag ProgressControlFlag) {
if len(m) == 0 {
cFlag = BreakProgress
} else if _, ok := m[addr.ObjectID.String()]; !ok {
cFlag = NextAddress
}
return
},
Token: token,
Bearer: bearer,
ExtendedHeaders: extHdrs,
},
Handler: func(_ multiaddr.Multiaddr, obj *object.Object) {
_, hashHeader := obj.LastHeader(object.HeaderType(object.HomoHashHdr))
if hashHeader == nil {
return
}
hashList = append(hashList, hashHeader.Value.(*object.Header_HomoHash).HomoHash)
res.ValidationDataSize += obj.SystemHeader.PayloadLength
delete(m, obj.SystemHeader.ID.String())
},
},
FullHeaders: true,
}); err != nil {
return nil, err
} else if len(m) > 0 {
return nil, ErrIncompleteSGInfo
}
res.ValidationHash, err = hash.Concat(hashList)
return res, err
}
// NewStorageGroupInfoReceiver constructs storagegroup.InfoReceiver from SelectiveContainerExecutor.
func NewStorageGroupInfoReceiver(p StorageGroupInfoReceiverParams) (storagegroup.InfoReceiver, error) {
switch {
case p.Logger == nil:
return nil, errors.Wrap(errEmptyLogger, locationFinderInstanceFailMsg)
case p.SelectiveContainerExecutor == nil:
return nil, errors.Wrap(errEmptyObjectsContainerHandler, locationFinderInstanceFailMsg)
}
return &sgInfoRecv{
executor: p.SelectiveContainerExecutor,
log: p.Logger,
}, nil
}

View file

@ -0,0 +1,657 @@
package implementations
import (
"context"
"io"
"sync"
"time"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-api-go/hash"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-api-go/service"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/transport"
"github.com/pkg/errors"
"go.uber.org/zap"
)
/*
File source code includes implementation of unified objects container handler.
Implementation provides the opportunity to perform any logic over object container distributed in network.
Implementation holds placement and object transport implementations in a black box.
Any special logic could be tuned through passing handle parameters.
NOTE: Although the implementation of the other interfaces via OCH is the same, they are still separated in order to avoid mess.
*/
type (
// SelectiveContainerExecutor is an interface the tool that performs
// object operations in container with preconditions.
SelectiveContainerExecutor interface {
Put(context.Context, *PutParams) error
Get(context.Context, *GetParams) error
Head(context.Context, *HeadParams) error
Search(context.Context, *SearchParams) error
RangeHash(context.Context, *RangeHashParams) error
}
// PutParams groups the parameters
// of selective object Put.
PutParams struct {
SelectiveParams
Object *object.Object
Handler func(multiaddr.Multiaddr, bool)
CopiesNumber uint32
}
// GetParams groups the parameters
// of selective object Get.
GetParams struct {
SelectiveParams
Handler func(multiaddr.Multiaddr, *object.Object)
}
// HeadParams groups the parameters
// of selective object Head.
HeadParams struct {
GetParams
FullHeaders bool
}
// SearchParams groups the parameters
// of selective object Search.
SearchParams struct {
SelectiveParams
SearchCID refs.CID
SearchQuery []byte
Handler func(multiaddr.Multiaddr, []refs.Address)
}
// RangeHashParams groups the parameters
// of selective object GetRangeHash.
RangeHashParams struct {
SelectiveParams
Ranges []object.Range
Salt []byte
Handler func(multiaddr.Multiaddr, []hash.Hash)
}
// SelectiveParams groups the parameters of
// the execution of selective container operation.
SelectiveParams struct {
/* Should be set to true only if service under object transport implementations is served on localhost. */
ServeLocal bool
/* Raw option of the request */
Raw bool
/* TTL for object transport. All transport operations inherit same value. */
TTL uint32
/* Required ID of processing container. If empty or not set, an error is returned. */
CID
/* List of nodes selected for processing. If not specified => nodes will be selected during. */
Nodes []multiaddr.Multiaddr
/*
Next two parameters provide the opportunity to process selective objects in container.
At least on of non-empty IDList or Query is required, an error is returned otherwise.
*/
/* List of objects to process (overlaps query). */
IDList []refs.ObjectID
/* If no objects is indicated, query is used for selection. */
Query []byte
/*
If function provided, it is called after every successful operation.
True result breaks operation performing.
*/
Breaker func(refs.Address) ProgressControlFlag
/* Public session token */
Token service.SessionToken
/* Bearer token */
Bearer service.BearerToken
/* Extended headers */
ExtendedHeaders []service.ExtendedHeader
}
// ProgressControlFlag is an enumeration of progress control flags.
ProgressControlFlag int
// ObjectContainerHandlerParams grops the parameters of SelectiveContainerExecutor constructor.
ObjectContainerHandlerParams struct {
NodeLister ContainerNodesLister
Executor ContainerTraverseExecutor
*zap.Logger
}
simpleTraverser struct {
*sync.Once
list []multiaddr.Multiaddr
}
selectiveCnrExec struct {
cnl ContainerNodesLister
Executor ContainerTraverseExecutor
log *zap.Logger
}
metaInfo struct {
ttl uint32
raw bool
rt object.RequestType
token service.SessionToken
bearer service.BearerToken
extHdrs []service.ExtendedHeader
}
putInfo struct {
metaInfo
obj *object.Object
cn uint32
}
getInfo struct {
metaInfo
addr Address
raw bool
}
headInfo struct {
getInfo
fullHdr bool
}
searchInfo struct {
metaInfo
cid CID
query []byte
}
rangeHashInfo struct {
metaInfo
addr Address
ranges []object.Range
salt []byte
}
execItems struct {
params SelectiveParams
metaConstructor func(addr Address) transport.MetaInfo
handler transport.ResultHandler
}
searchTarget struct {
list []refs.Address
}
// ContainerTraverseExecutor is an interface of
// object operation executor with container traversing.
ContainerTraverseExecutor interface {
Execute(context.Context, TraverseParams)
}
// TraverseParams groups the parameters of container traversing.
TraverseParams struct {
TransportInfo transport.MetaInfo
Handler transport.ResultHandler
Traverser Traverser
WorkerPool WorkerPool
ExecutionInterceptor func(context.Context, multiaddr.Multiaddr) bool
}
// WorkerPool is an interface of go-routine pool
WorkerPool interface {
Submit(func()) error
}
// Traverser is an interface of container traverser.
Traverser interface {
Next(context.Context) []multiaddr.Multiaddr
}
cnrTraverseExec struct {
transport transport.ObjectTransport
}
singleRoutinePool struct{}
emptyReader struct{}
)
const (
_ ProgressControlFlag = iota
// NextAddress is a ProgressControlFlag of to go to the next address of the object.
NextAddress
// NextNode is a ProgressControlFlag of to go to the next node.
NextNode
// BreakProgress is a ProgressControlFlag to interrupt the execution.
BreakProgress
)
const (
instanceFailMsg = "could not create container objects collector"
errEmptyLogger = internal.Error("empty logger")
errEmptyNodeLister = internal.Error("empty container node lister")
errEmptyTraverseExecutor = internal.Error("empty container traverse executor")
errSelectiveParams = internal.Error("neither ID list nor query provided")
)
var errNilObjectTransport = errors.New("object transport is nil")
func (s *selectiveCnrExec) Put(ctx context.Context, p *PutParams) error {
meta := &putInfo{
metaInfo: metaInfo{
ttl: p.TTL,
rt: object.RequestPut,
raw: p.Raw,
token: p.Token,
bearer: p.Bearer,
extHdrs: p.ExtendedHeaders,
},
obj: p.Object,
cn: p.CopiesNumber,
}
return s.exec(ctx, &execItems{
params: p.SelectiveParams,
metaConstructor: func(Address) transport.MetaInfo { return meta },
handler: p,
})
}
func (s *selectiveCnrExec) Get(ctx context.Context, p *GetParams) error {
return s.exec(ctx, &execItems{
params: p.SelectiveParams,
metaConstructor: func(addr Address) transport.MetaInfo {
return &getInfo{
metaInfo: metaInfo{
ttl: p.TTL,
rt: object.RequestGet,
raw: p.Raw,
token: p.Token,
bearer: p.Bearer,
extHdrs: p.ExtendedHeaders,
},
addr: addr,
raw: p.Raw,
}
},
handler: p,
})
}
func (s *selectiveCnrExec) Head(ctx context.Context, p *HeadParams) error {
return s.exec(ctx, &execItems{
params: p.SelectiveParams,
metaConstructor: func(addr Address) transport.MetaInfo {
return &headInfo{
getInfo: getInfo{
metaInfo: metaInfo{
ttl: p.TTL,
rt: object.RequestHead,
raw: p.Raw,
token: p.Token,
bearer: p.Bearer,
extHdrs: p.ExtendedHeaders,
},
addr: addr,
raw: p.Raw,
},
fullHdr: p.FullHeaders,
}
},
handler: p,
})
}
func (s *selectiveCnrExec) Search(ctx context.Context, p *SearchParams) error {
return s.exec(ctx, &execItems{
params: p.SelectiveParams,
metaConstructor: func(Address) transport.MetaInfo {
return &searchInfo{
metaInfo: metaInfo{
ttl: p.TTL,
rt: object.RequestSearch,
raw: p.Raw,
token: p.Token,
bearer: p.Bearer,
extHdrs: p.ExtendedHeaders,
},
cid: p.SearchCID,
query: p.SearchQuery,
}
},
handler: p,
})
}
func (s *selectiveCnrExec) RangeHash(ctx context.Context, p *RangeHashParams) error {
return s.exec(ctx, &execItems{
params: p.SelectiveParams,
metaConstructor: func(addr Address) transport.MetaInfo {
return &rangeHashInfo{
metaInfo: metaInfo{
ttl: p.TTL,
rt: object.RequestRangeHash,
raw: p.Raw,
token: p.Token,
bearer: p.Bearer,
extHdrs: p.ExtendedHeaders,
},
addr: addr,
ranges: p.Ranges,
salt: p.Salt,
}
},
handler: p,
})
}
func (s *selectiveCnrExec) exec(ctx context.Context, p *execItems) error {
if err := p.params.validate(); err != nil {
return err
}
nodes, err := s.prepareNodes(ctx, &p.params)
if err != nil {
return err
}
loop:
for i := range nodes {
addrList := s.prepareAddrList(ctx, &p.params, nodes[i])
if len(addrList) == 0 {
continue
}
for j := range addrList {
if p.params.Breaker != nil {
switch cFlag := p.params.Breaker(addrList[j]); cFlag {
case NextAddress:
continue
case NextNode:
continue loop
case BreakProgress:
break loop
}
}
s.Executor.Execute(ctx, TraverseParams{
TransportInfo: p.metaConstructor(addrList[j]),
Handler: p.handler,
Traverser: newSimpleTraverser(nodes[i]),
})
}
}
return nil
}
func (s *SelectiveParams) validate() error {
switch {
case len(s.IDList) == 0 && len(s.Query) == 0:
return errSelectiveParams
default:
return nil
}
}
func (s *selectiveCnrExec) prepareNodes(ctx context.Context, p *SelectiveParams) ([]multiaddr.Multiaddr, error) {
if len(p.Nodes) > 0 {
return p.Nodes, nil
}
// If node serves Object transport service on localhost => pass single empty node
if p.ServeLocal {
// all transport implementations will use localhost by default
return []multiaddr.Multiaddr{nil}, nil
}
// Otherwise use container nodes
return s.cnl.ContainerNodes(ctx, p.CID)
}
func (s *selectiveCnrExec) prepareAddrList(ctx context.Context, p *SelectiveParams, node multiaddr.Multiaddr) []refs.Address {
var (
addrList []Address
l = len(p.IDList)
)
if l > 0 {
addrList = make([]Address, 0, l)
for i := range p.IDList {
addrList = append(addrList, Address{CID: p.CID, ObjectID: p.IDList[i]})
}
return addrList
}
handler := new(searchTarget)
s.Executor.Execute(ctx, TraverseParams{
TransportInfo: &searchInfo{
metaInfo: metaInfo{
ttl: p.TTL,
rt: object.RequestSearch,
raw: p.Raw,
token: p.Token,
bearer: p.Bearer,
extHdrs: p.ExtendedHeaders,
},
cid: p.CID,
query: p.Query,
},
Handler: handler,
Traverser: newSimpleTraverser(node),
})
return handler.list
}
func newSimpleTraverser(list ...multiaddr.Multiaddr) Traverser {
return &simpleTraverser{
Once: new(sync.Once),
list: list,
}
}
func (s *simpleTraverser) Next(context.Context) (res []multiaddr.Multiaddr) {
s.Do(func() {
res = s.list
})
return
}
func (s metaInfo) GetTTL() uint32 { return s.ttl }
func (s metaInfo) GetTimeout() time.Duration { return 0 }
func (s metaInfo) GetRaw() bool { return s.raw }
func (s metaInfo) Type() object.RequestType { return s.rt }
func (s metaInfo) GetSessionToken() service.SessionToken { return s.token }
func (s metaInfo) GetBearerToken() service.BearerToken { return s.bearer }
func (s metaInfo) ExtendedHeaders() []service.ExtendedHeader { return s.extHdrs }
func (s *putInfo) GetHead() *object.Object { return s.obj }
func (s *putInfo) Payload() io.Reader { return new(emptyReader) }
func (*emptyReader) Read(p []byte) (int, error) { return 0, io.EOF }
func (s *putInfo) CopiesNumber() uint32 {
return s.cn
}
func (s *getInfo) GetAddress() refs.Address { return s.addr }
func (s *getInfo) Raw() bool { return s.raw }
func (s *headInfo) GetFullHeaders() bool { return s.fullHdr }
func (s *searchInfo) GetCID() refs.CID { return s.cid }
func (s *searchInfo) GetQuery() []byte { return s.query }
func (s *rangeHashInfo) GetAddress() refs.Address { return s.addr }
func (s *rangeHashInfo) GetRanges() []object.Range { return s.ranges }
func (s *rangeHashInfo) GetSalt() []byte { return s.salt }
func (s *searchTarget) HandleResult(_ context.Context, _ multiaddr.Multiaddr, r interface{}, e error) {
if e == nil {
s.list = append(s.list, r.([]refs.Address)...)
}
}
// HandleResult calls Handler with:
// - Multiaddr with argument value;
// - error equality to nil.
func (s *PutParams) HandleResult(_ context.Context, node multiaddr.Multiaddr, _ interface{}, e error) {
s.Handler(node, e == nil)
}
// HandleResult calls Handler if error argument is nil with:
// - Multiaddr with argument value;
// - result casted to an Object pointer.
func (s *GetParams) HandleResult(_ context.Context, node multiaddr.Multiaddr, r interface{}, e error) {
if e == nil {
s.Handler(node, r.(*object.Object))
}
}
// HandleResult calls Handler if error argument is nil with:
// - Multiaddr with argument value;
// - result casted to Address slice.
func (s *SearchParams) HandleResult(_ context.Context, node multiaddr.Multiaddr, r interface{}, e error) {
if e == nil {
s.Handler(node, r.([]refs.Address))
}
}
// HandleResult calls Handler if error argument is nil with:
// - Multiaddr with argument value;
// - result casted to Hash slice.
func (s *RangeHashParams) HandleResult(_ context.Context, node multiaddr.Multiaddr, r interface{}, e error) {
if e == nil {
s.Handler(node, r.([]hash.Hash))
}
}
func (s *cnrTraverseExec) Execute(ctx context.Context, p TraverseParams) {
if p.WorkerPool == nil {
p.WorkerPool = new(singleRoutinePool)
}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
wg := new(sync.WaitGroup)
for {
select {
case <-ctx.Done():
return
default:
}
nodes := p.Traverser.Next(ctx)
if len(nodes) == 0 {
break
}
for i := range nodes {
node := nodes[i]
wg.Add(1)
if err := p.WorkerPool.Submit(func() {
defer wg.Done()
if p.ExecutionInterceptor != nil && p.ExecutionInterceptor(ctx, node) {
return
}
s.transport.Transport(ctx, transport.ObjectTransportParams{
TransportInfo: p.TransportInfo,
TargetNode: node,
ResultHandler: p.Handler,
})
}); err != nil {
wg.Done()
}
}
wg.Wait()
}
}
func (*singleRoutinePool) Submit(fn func()) error {
fn()
return nil
}
// NewObjectContainerHandler is a SelectiveContainerExecutor constructor.
func NewObjectContainerHandler(p ObjectContainerHandlerParams) (SelectiveContainerExecutor, error) {
switch {
case p.Executor == nil:
return nil, errors.Wrap(errEmptyTraverseExecutor, instanceFailMsg)
case p.Logger == nil:
return nil, errors.Wrap(errEmptyLogger, instanceFailMsg)
case p.NodeLister == nil:
return nil, errors.Wrap(errEmptyNodeLister, instanceFailMsg)
}
return &selectiveCnrExec{
cnl: p.NodeLister,
Executor: p.Executor,
log: p.Logger,
}, nil
}
// NewContainerTraverseExecutor is a ContainerTraverseExecutor executor.
func NewContainerTraverseExecutor(t transport.ObjectTransport) (ContainerTraverseExecutor, error) {
if t == nil {
return nil, errNilObjectTransport
}
return &cnrTraverseExec{transport: t}, nil
}

View file

@ -0,0 +1,405 @@
package implementations
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/sha256"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-api-go/hash"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-api-go/service"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/nspcc-dev/neofs-node/lib/localstore"
"github.com/nspcc-dev/neofs-node/lib/objutil"
"github.com/nspcc-dev/neofs-node/lib/rand"
"github.com/nspcc-dev/neofs-node/lib/replication"
"github.com/pkg/errors"
"go.uber.org/zap"
)
type (
objectValidator struct {
as AddressStore
ls localstore.Localstore
executor SelectiveContainerExecutor
log *zap.Logger
saltSize int
maxRngSize uint64
rangeCount int
sltr Salitor
verifier objutil.Verifier
}
// Salitor is a salting data function.
Salitor func(data, salt []byte) []byte
// ObjectValidatorParams groups th
ObjectValidatorParams struct {
AddressStore AddressStore
Localstore localstore.Localstore
SelectiveContainerExecutor SelectiveContainerExecutor
Logger *zap.Logger
Salitor Salitor
SaltSize int
MaxPayloadRangeSize uint64
PayloadRangeCount int
Verifier objutil.Verifier
}
localHeadIntegrityVerifier struct {
keyVerifier core.OwnerKeyVerifier
}
payloadVerifier struct {
}
localIntegrityVerifier struct {
headVerifier objutil.Verifier
payloadVerifier objutil.Verifier
}
)
const (
objectValidatorInstanceFailMsg = "could not create object validator"
errEmptyLocalstore = internal.Error("empty local storage")
errEmptyObjectVerifier = internal.Error("empty object verifier")
defaultSaltSize = 64 // bytes
defaultPayloadRangeCount = 3
defaultMaxPayloadRangeSize = 64
)
const (
errBrokenHeaderStructure = internal.Error("broken header structure")
errMissingPayloadChecksumHeader = internal.Error("missing payload checksum header")
errWrongPayloadChecksum = internal.Error("wrong payload checksum")
)
func (s *objectValidator) Verify(ctx context.Context, params *replication.ObjectVerificationParams) bool {
selfAddr, err := s.as.SelfAddr()
if err != nil {
s.log.Debug("receive self address failure", zap.Error(err))
return false
}
if params.Node == nil || params.Node.Equal(selfAddr) {
return s.verifyLocal(ctx, params.Address)
}
return s.verifyRemote(ctx, params)
}
func (s *objectValidator) verifyLocal(ctx context.Context, addr Address) bool {
var (
err error
obj *Object
)
if obj, err = s.ls.Get(addr); err != nil {
s.log.Debug("get local meta information failure", zap.Error(err))
return false
} else if err = s.verifier.Verify(ctx, obj); err != nil {
s.log.Debug("integrity check failure", zap.Error(err))
}
return err == nil
}
func (s *objectValidator) verifyRemote(ctx context.Context, params *replication.ObjectVerificationParams) bool {
var (
receivedObj *Object
valid bool
)
defer func() {
if params.Handler != nil && receivedObj != nil {
params.Handler(valid, receivedObj)
}
}()
p := &HeadParams{
GetParams: GetParams{
SelectiveParams: SelectiveParams{
CID: params.CID,
Nodes: []multiaddr.Multiaddr{params.Node},
TTL: service.NonForwardingTTL,
IDList: []ObjectID{params.ObjectID},
Raw: true,
},
Handler: func(_ multiaddr.Multiaddr, obj *object.Object) {
receivedObj = obj
valid = s.verifier.Verify(ctx, obj) == nil
},
},
FullHeaders: true,
}
if err := s.executor.Head(ctx, p); err != nil || !valid {
return false
} else if receivedObj.SystemHeader.PayloadLength <= 0 || receivedObj.IsLinking() {
return true
}
if !params.LocalInvalid {
has, err := s.ls.Has(params.Address)
if err == nil && has {
obj, err := s.ls.Get(params.Address)
if err == nil {
return s.verifyThroughHashes(ctx, obj, params.Node)
}
}
}
valid = false
_ = s.executor.Get(ctx, &p.GetParams)
return valid
}
func (s *objectValidator) verifyThroughHashes(ctx context.Context, obj *Object, node multiaddr.Multiaddr) (valid bool) {
var (
salt = generateSalt(s.saltSize)
rngs = generateRanges(obj.SystemHeader.PayloadLength, s.maxRngSize, s.rangeCount)
)
_ = s.executor.RangeHash(ctx, &RangeHashParams{
SelectiveParams: SelectiveParams{
CID: obj.SystemHeader.CID,
Nodes: []multiaddr.Multiaddr{node},
TTL: service.NonForwardingTTL,
IDList: []ObjectID{obj.SystemHeader.ID},
},
Ranges: rngs,
Salt: salt,
Handler: func(node multiaddr.Multiaddr, hashes []hash.Hash) {
valid = compareHashes(s.sltr, obj.Payload, salt, rngs, hashes)
},
})
return
}
func compareHashes(sltr Salitor, payload, salt []byte, rngs []object.Range, hashes []hash.Hash) bool {
if len(rngs) != len(hashes) {
return false
}
for i := range rngs {
saltPayloadPart := sltr(payload[rngs[i].Offset:rngs[i].Offset+rngs[i].Length], salt)
if !hashes[i].Equal(hash.Sum(saltPayloadPart)) {
return false
}
}
return true
}
func generateRanges(payloadSize, maxRangeSize uint64, count int) []object.Range {
res := make([]object.Range, count)
l := min(payloadSize, maxRangeSize)
for i := 0; i < count; i++ {
res[i].Length = l
res[i].Offset = rand.Uint64(rand.New(), int64(payloadSize-l))
}
return res
}
func min(a, b uint64) uint64 {
if a < b {
return a
}
return b
}
func generateSalt(saltSize int) []byte {
salt := make([]byte, saltSize)
if _, err := rand.Read(salt); err != nil {
return nil
}
return salt
}
// NewObjectValidator constructs universal replication.ObjectVerifier.
func NewObjectValidator(p *ObjectValidatorParams) (replication.ObjectVerifier, error) {
switch {
case p.Logger == nil:
return nil, errors.Wrap(errEmptyLogger, objectValidatorInstanceFailMsg)
case p.AddressStore == nil:
return nil, errors.Wrap(errEmptyAddressStore, objectValidatorInstanceFailMsg)
case p.Localstore == nil:
return nil, errors.Wrap(errEmptyLocalstore, objectValidatorInstanceFailMsg)
case p.Verifier == nil:
return nil, errors.Wrap(errEmptyObjectVerifier, objectValidatorInstanceFailMsg)
}
if p.SaltSize <= 0 {
p.SaltSize = defaultSaltSize
}
if p.PayloadRangeCount <= 0 {
p.PayloadRangeCount = defaultPayloadRangeCount
}
if p.MaxPayloadRangeSize <= 0 {
p.MaxPayloadRangeSize = defaultMaxPayloadRangeSize
}
return &objectValidator{
as: p.AddressStore,
ls: p.Localstore,
executor: p.SelectiveContainerExecutor,
log: p.Logger,
saltSize: p.SaltSize,
maxRngSize: p.MaxPayloadRangeSize,
rangeCount: p.PayloadRangeCount,
sltr: p.Salitor,
verifier: p.Verifier,
}, nil
}
// NewLocalHeadIntegrityVerifier constructs local object head verifier and returns objutil.Verifier interface.
func NewLocalHeadIntegrityVerifier(keyVerifier core.OwnerKeyVerifier) (objutil.Verifier, error) {
if keyVerifier == nil {
return nil, core.ErrNilOwnerKeyVerifier
}
return &localHeadIntegrityVerifier{
keyVerifier: keyVerifier,
}, nil
}
// NewLocalIntegrityVerifier constructs local object verifier and returns objutil.Verifier interface.
func NewLocalIntegrityVerifier(keyVerifier core.OwnerKeyVerifier) (objutil.Verifier, error) {
if keyVerifier == nil {
return nil, core.ErrNilOwnerKeyVerifier
}
return &localIntegrityVerifier{
headVerifier: &localHeadIntegrityVerifier{
keyVerifier: keyVerifier,
},
payloadVerifier: new(payloadVerifier),
}, nil
}
// NewPayloadVerifier constructs object payload verifier and returns objutil.Verifier.
func NewPayloadVerifier() objutil.Verifier {
return new(payloadVerifier)
}
type hdrOwnerKeyContainer struct {
owner refs.OwnerID
key []byte
}
func (s hdrOwnerKeyContainer) GetOwnerID() refs.OwnerID {
return s.owner
}
func (s hdrOwnerKeyContainer) GetOwnerKey() []byte {
return s.key
}
func (s *localHeadIntegrityVerifier) Verify(ctx context.Context, obj *Object) error {
var (
checkKey *ecdsa.PublicKey
ownerKeyCnr core.OwnerKeyContainer
)
if _, h := obj.LastHeader(object.HeaderType(object.TokenHdr)); h != nil {
token := h.GetValue().(*object.Header_Token).Token
if err := service.VerifySignatureWithKey(
crypto.UnmarshalPublicKey(token.GetOwnerKey()),
service.NewVerifiedSessionToken(token),
); err != nil {
return err
}
ownerKeyCnr = token
checkKey = crypto.UnmarshalPublicKey(token.GetSessionKey())
} else if _, h := obj.LastHeader(object.HeaderType(object.PublicKeyHdr)); h != nil {
pkHdr := h.GetValue().(*object.Header_PublicKey)
if pkHdr != nil && pkHdr.PublicKey != nil {
val := pkHdr.PublicKey.GetValue()
ownerKeyCnr = &hdrOwnerKeyContainer{
owner: obj.GetSystemHeader().OwnerID,
key: val,
}
checkKey = crypto.UnmarshalPublicKey(val)
}
}
if ownerKeyCnr == nil {
return core.ErrNilOwnerKeyContainer
} else if err := s.keyVerifier.VerifyKey(ctx, ownerKeyCnr); err != nil {
return err
}
return verifyObjectIntegrity(obj, checkKey)
}
// verifyObjectIntegrity verifies integrity of object header.
// Returns error if object
// - does not contains integrity header;
// - integrity header is not a last header in object;
// - integrity header signature is broken.
func verifyObjectIntegrity(obj *Object, key *ecdsa.PublicKey) error {
n, h := obj.LastHeader(object.HeaderType(object.IntegrityHdr))
if l := len(obj.Headers); l <= 0 || n != l-1 {
return errBrokenHeaderStructure
}
integrityHdr := h.Value.(*object.Header_Integrity).Integrity
if integrityHdr == nil {
return errBrokenHeaderStructure
}
data, err := objutil.MarshalHeaders(obj, n)
if err != nil {
return err
}
hdrChecksum := sha256.Sum256(data)
return crypto.Verify(key, hdrChecksum[:], integrityHdr.ChecksumSignature)
}
func (s *payloadVerifier) Verify(_ context.Context, obj *Object) error {
if _, h := obj.LastHeader(object.HeaderType(object.PayloadChecksumHdr)); h == nil {
return errMissingPayloadChecksumHeader
} else if checksum := sha256.Sum256(obj.Payload); !bytes.Equal(
checksum[:],
h.Value.(*object.Header_PayloadChecksum).PayloadChecksum,
) {
return errWrongPayloadChecksum
}
return nil
}
func (s *localIntegrityVerifier) Verify(ctx context.Context, obj *Object) error {
if err := s.headVerifier.Verify(ctx, obj); err != nil {
return err
}
return s.payloadVerifier.Verify(ctx, obj)
}

View file

@ -0,0 +1,273 @@
package implementations
import (
"context"
"crypto/ecdsa"
"crypto/sha256"
"math/rand"
"testing"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-api-go/service"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/nspcc-dev/neofs-node/lib/localstore"
"github.com/nspcc-dev/neofs-node/lib/objutil"
"github.com/nspcc-dev/neofs-node/lib/test"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
type testEntity struct {
err error
}
func (s *testEntity) Verify(context.Context, *object.Object) error { return s.err }
func (s *testEntity) SelfAddr() (multiaddr.Multiaddr, error) { panic("implement me") }
func (s *testEntity) Put(context.Context, *localstore.Object) error { panic("implement me") }
func (s *testEntity) Get(localstore.Address) (*localstore.Object, error) { panic("implement me") }
func (s *testEntity) Del(localstore.Address) error { panic("implement me") }
func (s *testEntity) Meta(localstore.Address) (*localstore.ObjectMeta, error) { panic("implement me") }
func (s *testEntity) Has(localstore.Address) (bool, error) { panic("implement me") }
func (s *testEntity) ObjectsCount() (uint64, error) { panic("implement me") }
func (s *testEntity) Size() int64 { panic("implement me") }
func (s *testEntity) Iterate(localstore.FilterPipeline, localstore.MetaHandler) error {
panic("implement me")
}
func (s *testEntity) PRead(ctx context.Context, addr refs.Address, rng object.Range) ([]byte, error) {
panic("implement me")
}
func (s *testEntity) VerifyKey(context.Context, core.OwnerKeyContainer) error {
return s.err
}
func TestNewObjectValidator(t *testing.T) {
validParams := ObjectValidatorParams{
Logger: zap.L(),
AddressStore: new(testEntity),
Localstore: new(testEntity),
Verifier: new(testEntity),
}
t.Run("valid params", func(t *testing.T) {
s, err := NewObjectValidator(&validParams)
require.NoError(t, err)
require.NotNil(t, s)
})
t.Run("fail on empty local storage", func(t *testing.T) {
p := validParams
p.Localstore = nil
_, err := NewObjectValidator(&p)
require.EqualError(t, err, errors.Wrap(errEmptyLocalstore, objectValidatorInstanceFailMsg).Error())
})
t.Run("fail on empty logger", func(t *testing.T) {
p := validParams
p.Logger = nil
_, err := NewObjectValidator(&p)
require.EqualError(t, err, errors.Wrap(errEmptyLogger, objectValidatorInstanceFailMsg).Error())
})
}
func TestNewLocalIntegrityVerifier(t *testing.T) {
var (
err error
verifier objutil.Verifier
keyVerifier = new(testEntity)
)
_, err = NewLocalHeadIntegrityVerifier(nil)
require.EqualError(t, err, core.ErrNilOwnerKeyVerifier.Error())
_, err = NewLocalIntegrityVerifier(nil)
require.EqualError(t, err, core.ErrNilOwnerKeyVerifier.Error())
verifier, err = NewLocalHeadIntegrityVerifier(keyVerifier)
require.NoError(t, err)
require.NotNil(t, verifier)
verifier, err = NewLocalIntegrityVerifier(keyVerifier)
require.NoError(t, err)
require.NotNil(t, verifier)
}
func TestLocalHeadIntegrityVerifier_Verify(t *testing.T) {
var (
ctx = context.TODO()
ownerPrivateKey = test.DecodeKey(0)
ownerPublicKey = &ownerPrivateKey.PublicKey
sessionPrivateKey = test.DecodeKey(1)
sessionPublicKey = &sessionPrivateKey.PublicKey
)
ownerID, err := refs.NewOwnerID(ownerPublicKey)
require.NoError(t, err)
s, err := NewLocalIntegrityVerifier(core.NewNeoKeyVerifier())
require.NoError(t, err)
okItems := []func() *Object{
// correct object w/ session token
func() *Object {
token := new(service.Token)
token.SetOwnerID(ownerID)
token.SetSessionKey(crypto.MarshalPublicKey(sessionPublicKey))
require.NoError(t,
service.AddSignatureWithKey(
ownerPrivateKey,
service.NewSignedSessionToken(token),
),
)
obj := new(Object)
obj.AddHeader(&object.Header{
Value: &object.Header_Token{
Token: token,
},
})
obj.SetPayload([]byte{1, 2, 3})
addPayloadChecksum(obj)
addHeadersChecksum(t, obj, sessionPrivateKey)
return obj
},
// correct object w/o session token
func() *Object {
obj := new(Object)
obj.SystemHeader.OwnerID = ownerID
obj.SetPayload([]byte{1, 2, 3})
addPayloadChecksum(obj)
obj.AddHeader(&object.Header{
Value: &object.Header_PublicKey{
PublicKey: &object.PublicKey{
Value: crypto.MarshalPublicKey(ownerPublicKey),
},
},
})
addHeadersChecksum(t, obj, ownerPrivateKey)
return obj
},
}
failItems := []func() *Object{}
for _, item := range okItems {
require.NoError(t, s.Verify(ctx, item()))
}
for _, item := range failItems {
require.Error(t, s.Verify(ctx, item()))
}
}
func addPayloadChecksum(obj *Object) {
payloadChecksum := sha256.Sum256(obj.GetPayload())
obj.AddHeader(&object.Header{
Value: &object.Header_PayloadChecksum{
PayloadChecksum: payloadChecksum[:],
},
})
}
func addHeadersChecksum(t *testing.T, obj *Object, key *ecdsa.PrivateKey) {
headersData, err := objutil.MarshalHeaders(obj, len(obj.Headers))
require.NoError(t, err)
headersChecksum := sha256.Sum256(headersData)
integrityHdr := new(object.IntegrityHeader)
integrityHdr.SetHeadersChecksum(headersChecksum[:])
require.NoError(t, service.AddSignatureWithKey(key, integrityHdr))
obj.AddHeader(&object.Header{
Value: &object.Header_Integrity{
Integrity: integrityHdr,
},
})
}
func TestPayloadVerifier_Verify(t *testing.T) {
ctx := context.TODO()
verifier := new(payloadVerifier)
t.Run("missing header", func(t *testing.T) {
obj := new(Object)
require.EqualError(t, verifier.Verify(ctx, obj), errMissingPayloadChecksumHeader.Error())
})
t.Run("correct result", func(t *testing.T) {
payload := testData(t, 10)
cs := sha256.Sum256(payload)
hdr := &object.Header_PayloadChecksum{PayloadChecksum: cs[:]}
obj := &Object{
Headers: []object.Header{{Value: hdr}},
Payload: payload,
}
require.NoError(t, verifier.Verify(ctx, obj))
hdr.PayloadChecksum[0]++
require.EqualError(t, verifier.Verify(ctx, obj), errWrongPayloadChecksum.Error())
hdr.PayloadChecksum[0]--
obj.Payload[0]++
require.EqualError(t, verifier.Verify(ctx, obj), errWrongPayloadChecksum.Error())
})
}
func TestLocalIntegrityVerifier_Verify(t *testing.T) {
ctx := context.TODO()
obj := new(Object)
t.Run("head verification failure", func(t *testing.T) {
hErr := internal.Error("test error for head verifier")
s := &localIntegrityVerifier{
headVerifier: &testEntity{
err: hErr, // force head verifier to return hErr
},
}
require.EqualError(t, s.Verify(ctx, obj), hErr.Error())
})
t.Run("correct result", func(t *testing.T) {
pErr := internal.Error("test error for payload verifier")
s := &localIntegrityVerifier{
headVerifier: new(testEntity),
payloadVerifier: &testEntity{
err: pErr, // force payload verifier to return hErr
},
}
require.EqualError(t, s.Verify(ctx, obj), pErr.Error())
})
}
// testData returns size bytes of random data.
func testData(t *testing.T, size int) []byte {
res := make([]byte, size)
_, err := rand.Read(res)
require.NoError(t, err)
return res
}
// TODO: write functionality tests

17
lib/ir/info.go Normal file
View file

@ -0,0 +1,17 @@
package ir
// Info is a structure that groups the information
// about inner ring.
type Info struct {
nodes []Node
}
// SetNodes is an IR node list setter.
func (s *Info) SetNodes(v []Node) {
s.nodes = v
}
// Nodes is an IR node list getter.
func (s Info) Nodes() []Node {
return s.nodes
}

25
lib/ir/info_test.go Normal file
View file

@ -0,0 +1,25 @@
package ir
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestInfo(t *testing.T) {
s := Info{}
n1 := Node{}
n1.SetKey([]byte{1, 2, 3})
n2 := Node{}
n2.SetKey([]byte{4, 5, 6})
nodes := []Node{
n1,
n2,
}
s.SetNodes(nodes)
require.Equal(t, nodes, s.Nodes())
}

17
lib/ir/node.go Normal file
View file

@ -0,0 +1,17 @@
package ir
// Node is a structure that groups
// the information about IR node.
type Node struct {
key []byte
}
// SetKey is an IR node public key setter.
func (s *Node) SetKey(v []byte) {
s.key = v
}
// Key is an IR node public key getter.
func (s Node) Key() []byte {
return s.key
}

16
lib/ir/node_test.go Normal file
View file

@ -0,0 +1,16 @@
package ir
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestNode(t *testing.T) {
s := Node{}
key := []byte{1, 2, 3}
s.SetKey(key)
require.Equal(t, key, s.Key())
}

94
lib/ir/storage.go Normal file
View file

@ -0,0 +1,94 @@
package ir
import (
"bytes"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/pkg/errors"
)
// Storage is an interface of the storage of info about NeoFS IR.
type Storage interface {
GetIRInfo(GetInfoParams) (*GetInfoResult, error)
}
// GetInfoParams is a structure that groups the parameters
// for IR info receiving operation.
type GetInfoParams struct {
}
// GetInfoResult is a structure that groups
// values returned by IR info receiving operation.
type GetInfoResult struct {
info Info
}
// ErrNilStorage is returned by functions that expect
// a non-nil Storage, but received nil.
const ErrNilStorage = internal.Error("inner ring storage is nil")
// SetInfo is an IR info setter.
func (s *GetInfoResult) SetInfo(v Info) {
s.info = v
}
// Info is an IR info getter.
func (s GetInfoResult) Info() Info {
return s.info
}
// BinaryKeyList returns the list of binary public key of IR nodes.
//
// If passed Storage is nil, ErrNilStorage returns.
func BinaryKeyList(storage Storage) ([][]byte, error) {
if storage == nil {
return nil, ErrNilStorage
}
// get IR info
getRes, err := storage.GetIRInfo(GetInfoParams{})
if err != nil {
return nil, errors.Wrap(
err,
"could not get information about IR",
)
}
nodes := getRes.Info().Nodes()
keys := make([][]byte, 0, len(nodes))
for i := range nodes {
keys = append(keys, nodes[i].Key())
}
return keys, nil
}
// IsInnerRingKey checks if the passed argument is the
// key of one of IR nodes.
//
// Uses BinaryKeyList function to receive the key list of IR nodes internally.
//
// If passed key slice is empty, crypto.ErrEmptyPublicKey returns immediately.
func IsInnerRingKey(storage Storage, key []byte) (bool, error) {
// check key emptiness
// TODO: summarize the void check to a full IR key-format check.
if len(key) == 0 {
return false, crypto.ErrEmptyPublicKey
}
irKeys, err := BinaryKeyList(storage)
if err != nil {
return false, err
}
for i := range irKeys {
if bytes.Equal(irKeys[i], key) {
return true, nil
}
}
return false, nil
}

101
lib/ir/storage_test.go Normal file
View file

@ -0,0 +1,101 @@
package ir
import (
"testing"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
)
type testInfoReceiver struct {
keys [][]byte
err error
}
func (s testInfoReceiver) GetIRInfo(GetInfoParams) (*GetInfoResult, error) {
if s.err != nil {
return nil, s.err
}
nodes := make([]Node, 0, len(s.keys))
for i := range s.keys {
node := Node{}
node.SetKey(s.keys[i])
nodes = append(nodes, node)
}
info := Info{}
info.SetNodes(nodes)
res := new(GetInfoResult)
res.SetInfo(info)
return res, nil
}
func (s *testInfoReceiver) addKey(key []byte) {
s.keys = append(s.keys, key)
}
func TestGetInfoResult(t *testing.T) {
s := GetInfoResult{}
info := Info{}
n := Node{}
n.SetKey([]byte{1, 2, 3})
info.SetNodes([]Node{
n,
})
s.SetInfo(info)
require.Equal(t, info, s.Info())
}
func TestIsInnerRingKey(t *testing.T) {
var (
res bool
err error
s = new(testInfoReceiver)
)
// empty public key
res, err = IsInnerRingKey(nil, nil)
require.EqualError(t, err, crypto.ErrEmptyPublicKey.Error())
key := []byte{1, 2, 3}
// nil Storage
res, err = IsInnerRingKey(nil, key)
require.EqualError(t, err, ErrNilStorage.Error())
// force Storage to return an error
s.err = errors.New("some error")
// Storage error
res, err = IsInnerRingKey(s, key)
require.EqualError(t, errors.Cause(err), s.err.Error())
// reset Storage error
s.err = nil
// IR keys don't contain key
s.addKey(append(key, 1))
res, err = IsInnerRingKey(s, key)
require.NoError(t, err)
require.False(t, res)
// IR keys contain key
s.addKey(key)
res, err = IsInnerRingKey(s, key)
require.NoError(t, err)
require.True(t, res)
}

35
lib/localstore/alias.go Normal file
View file

@ -0,0 +1,35 @@
package localstore
import (
"github.com/nspcc-dev/neofs-api-go/hash"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
)
// CID is a type alias of
// CID from refs package of neofs-api-go.
type CID = refs.CID
// SGID is a type alias of
// SGID from refs package of neofs-api-go.
type SGID = refs.ObjectID
// Header is a type alias of
// Header from object package of neofs-api-go.
type Header = object.Header
// Object is a type alias of
// Object from object package of neofs-api-go.
type Object = object.Object
// ObjectID is a type alias of
// ObjectID from refs package of neofs-api-go.
type ObjectID = refs.ObjectID
// Address is a type alias of
// Address from refs package of neofs-api-go.
type Address = refs.Address
// Hash is a type alias of
// Hash from hash package of neofs-api-go.
type Hash = hash.Hash

38
lib/localstore/del.go Normal file
View file

@ -0,0 +1,38 @@
package localstore
import (
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-node/lib/metrics"
"github.com/pkg/errors"
"go.uber.org/zap"
)
func (l *localstore) Del(key refs.Address) error {
k, err := key.Hash()
if err != nil {
return errors.Wrap(err, "Localstore Del failed on key.Marshal")
}
// try to fetch object for metrics
obj, err := l.Get(key)
if err != nil {
l.log.Warn("localstore Del failed on localstore.Get", zap.Error(err))
}
if err := l.blobBucket.Del(k); err != nil {
l.log.Warn("Localstore Del failed on BlobBucket.Del", zap.Error(err))
}
if err := l.metaBucket.Del(k); err != nil {
return errors.Wrap(err, "Localstore Del failed on MetaBucket.Del")
}
if obj != nil {
l.col.UpdateContainer(
key.CID,
obj.SystemHeader.PayloadLength,
metrics.RemSpace)
}
return nil
}

306
lib/localstore/filter.go Normal file
View file

@ -0,0 +1,306 @@
package localstore
import (
"context"
"math"
"sort"
"sync"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/pkg/errors"
)
type (
// FilterCode is an enumeration of filter return codes.
FilterCode int
// PriorityFlag is an enumeration of priority flags.
PriorityFlag int
filterPipelineSet []FilterPipeline
// FilterFunc is a function that checks whether an ObjectMeta matches a specific criterion.
FilterFunc func(ctx context.Context, meta *ObjectMeta) *FilterResult
// FilterResult groups of ObjectMeta filter result values.
FilterResult struct {
c FilterCode
e error
}
// FilterPipeline is an interface of ObjectMeta filtering tool with sub-filters and priorities.
FilterPipeline interface {
Pass(ctx context.Context, meta *ObjectMeta) *FilterResult
PutSubFilter(params SubFilterParams) error
GetPriority() uint64
SetPriority(uint64)
GetName() string
}
// FilterParams groups the parameters of FilterPipeline constructor.
FilterParams struct {
Name string
Priority uint64
FilterFunc FilterFunc
}
// SubFilterParams groups the parameters of sub-filter registration.
SubFilterParams struct {
PriorityFlag
FilterPipeline
OnIgnore FilterCode
OnPass FilterCode
OnFail FilterCode
}
filterPipeline struct {
*sync.RWMutex
name string
pri uint64
filterFn FilterFunc
maxSubPri uint64
mSubResult map[string]map[FilterCode]FilterCode
subFilters []FilterPipeline
}
)
const (
// PriorityValue is a PriorityFlag of the sub-filter registration with GetPriority() value.
PriorityValue PriorityFlag = iota
// PriorityMax is a PriorityFlag of the sub-filter registration with maximum priority.
PriorityMax
// PriorityMin is a PriorityFlag of the sub-filter registration with minimum priority.
PriorityMin
)
const (
// CodeUndefined is a undefined FilterCode.
CodeUndefined FilterCode = iota
// CodePass is a FilterCode of filter passage.
CodePass
// CodeFail is a FilterCode of filter failure.
CodeFail
// CodeIgnore is a FilterCode of filter ignoring.
CodeIgnore
)
var (
rPass = &FilterResult{
c: CodePass,
}
rFail = &FilterResult{
c: CodeFail,
}
rIgnore = &FilterResult{
c: CodeIgnore,
}
rUndefined = &FilterResult{
c: CodeUndefined,
}
)
// ResultPass returns the FilterResult with CodePass code and nil error.
func ResultPass() *FilterResult {
return rPass
}
// ResultFail returns the FilterResult with CodeFail code and nil error.
func ResultFail() *FilterResult {
return rFail
}
// ResultIgnore returns the FilterResult with CodeIgnore code and nil error.
func ResultIgnore() *FilterResult {
return rIgnore
}
// ResultUndefined returns the FilterResult with CodeUndefined code and nil error.
func ResultUndefined() *FilterResult {
return rUndefined
}
// ResultWithError returns the FilterResult with passed code and error.
func ResultWithError(c FilterCode, e error) *FilterResult {
return &FilterResult{
e: e,
c: c,
}
}
// Code returns the filter result code.
func (s *FilterResult) Code() FilterCode {
return s.c
}
// Err returns the filter result error.
func (s *FilterResult) Err() error {
return s.e
}
func (f filterPipelineSet) Len() int { return len(f) }
func (f filterPipelineSet) Less(i, j int) bool { return f[i].GetPriority() > f[j].GetPriority() }
func (f filterPipelineSet) Swap(i, j int) { f[i], f[j] = f[j], f[i] }
func (r FilterCode) String() string {
switch r {
case CodePass:
return "PASSED"
case CodeFail:
return "FAILED"
case CodeIgnore:
return "IGNORED"
default:
return "UNDEFINED"
}
}
// NewFilter is a FilterPipeline constructor.
func NewFilter(p *FilterParams) FilterPipeline {
return &filterPipeline{
RWMutex: new(sync.RWMutex),
name: p.Name,
pri: p.Priority,
filterFn: p.FilterFunc,
mSubResult: make(map[string]map[FilterCode]FilterCode),
}
}
// AllPassIncludingFilter returns FilterPipeline with sub-filters composed from parameters.
// Result filter fails with CodeFail code if any of the sub-filters returns not a CodePass code.
func AllPassIncludingFilter(name string, params ...*FilterParams) (FilterPipeline, error) {
res := NewFilter(&FilterParams{
Name: name,
FilterFunc: SkippingFilterFunc,
})
for i := range params {
if err := res.PutSubFilter(SubFilterParams{
FilterPipeline: NewFilter(params[i]),
OnIgnore: CodeFail,
OnFail: CodeFail,
}); err != nil {
return nil, errors.Wrap(err, "could not create all pass including filter")
}
}
return res, nil
}
func (p *filterPipeline) Pass(ctx context.Context, meta *ObjectMeta) *FilterResult {
p.RLock()
defer p.RUnlock()
for i := range p.subFilters {
subResult := p.subFilters[i].Pass(ctx, meta)
subName := p.subFilters[i].GetName()
cSub := subResult.Code()
if cSub <= CodeUndefined {
return ResultUndefined()
}
if cFin := p.mSubResult[subName][cSub]; cFin != CodeIgnore {
return ResultWithError(cFin, subResult.Err())
}
}
if p.filterFn == nil {
return ResultUndefined()
}
return p.filterFn(ctx, meta)
}
func (p *filterPipeline) PutSubFilter(params SubFilterParams) error {
p.Lock()
defer p.Unlock()
if params.FilterPipeline == nil {
return internal.Error("could not put sub filter: empty filter pipeline")
}
name := params.FilterPipeline.GetName()
if _, ok := p.mSubResult[name]; ok {
return errors.Errorf("filter %s is already in pipeline %s", name, p.GetName())
}
if params.PriorityFlag != PriorityMin {
if pri := params.FilterPipeline.GetPriority(); pri < math.MaxUint64 {
params.FilterPipeline.SetPriority(pri + 1)
}
} else {
params.FilterPipeline.SetPriority(0)
}
switch pri := params.FilterPipeline.GetPriority(); params.PriorityFlag {
case PriorityMax:
if p.maxSubPri < math.MaxUint64 {
p.maxSubPri++
}
params.FilterPipeline.SetPriority(p.maxSubPri)
case PriorityValue:
if pri > p.maxSubPri {
p.maxSubPri = pri
}
}
if params.OnFail <= 0 {
params.OnFail = CodeIgnore
}
if params.OnIgnore <= 0 {
params.OnIgnore = CodeIgnore
}
if params.OnPass <= 0 {
params.OnPass = CodeIgnore
}
p.mSubResult[name] = map[FilterCode]FilterCode{
CodePass: params.OnPass,
CodeIgnore: params.OnIgnore,
CodeFail: params.OnFail,
}
p.subFilters = append(p.subFilters, params.FilterPipeline)
sort.Sort(filterPipelineSet(p.subFilters))
return nil
}
func (p *filterPipeline) GetPriority() uint64 {
p.RLock()
defer p.RUnlock()
return p.pri
}
func (p *filterPipeline) SetPriority(pri uint64) {
p.Lock()
p.pri = pri
p.Unlock()
}
func (p *filterPipeline) GetName() string {
p.RLock()
defer p.RUnlock()
if p.name == "" {
return "FILTER_UNNAMED"
}
return p.name
}

View file

@ -0,0 +1,39 @@
package localstore
import (
"context"
)
// SkippingFilterFunc is a FilterFunc that always returns result with
// CodePass code and nil error.
func SkippingFilterFunc(_ context.Context, _ *ObjectMeta) *FilterResult {
return ResultPass()
}
// ContainerFilterFunc returns a FilterFunc that returns:
// - result with CodePass code and nil error if CID of ObjectMeta if from the CID list;
// - result with CodeFail code an nil error otherwise.
func ContainerFilterFunc(cidList []CID) FilterFunc {
return func(_ context.Context, meta *ObjectMeta) *FilterResult {
for i := range cidList {
if meta.Object.SystemHeader.CID.Equal(cidList[i]) {
return ResultPass()
}
}
return ResultFail()
}
}
// StoredEarlierThanFilterFunc returns a FilterFunc that returns:
// - result with CodePass code and nil error if StoreEpoch is less that argument;
// - result with CodeFail code and nil error otherwise.
func StoredEarlierThanFilterFunc(epoch uint64) FilterFunc {
return func(_ context.Context, meta *ObjectMeta) *FilterResult {
if meta.StoreEpoch < epoch {
return ResultPass()
}
return ResultFail()
}
}

View file

@ -0,0 +1,38 @@
package localstore
import (
"context"
"testing"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/stretchr/testify/require"
)
func TestSkippingFilterFunc(t *testing.T) {
res := SkippingFilterFunc(context.TODO(), &ObjectMeta{})
require.Equal(t, CodePass, res.Code())
}
func TestFilterResult(t *testing.T) {
var (
r *FilterResult
c = CodePass
e = internal.Error("test error")
)
r = ResultPass()
require.Equal(t, CodePass, r.Code())
require.NoError(t, r.Err())
r = ResultFail()
require.Equal(t, CodeFail, r.Code())
require.NoError(t, r.Err())
r = ResultIgnore()
require.Equal(t, CodeIgnore, r.Code())
require.NoError(t, r.Err())
r = ResultWithError(c, e)
require.Equal(t, c, r.Code())
require.EqualError(t, r.Err(), e.Error())
}

30
lib/localstore/get.go Normal file
View file

@ -0,0 +1,30 @@
package localstore
import (
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/pkg/errors"
)
func (l *localstore) Get(key refs.Address) (*Object, error) {
var (
err error
k, v []byte
o = new(Object)
)
k, err = key.Hash()
if err != nil {
return nil, errors.Wrap(err, "Localstore Get failed on key.Marshal")
}
v, err = l.blobBucket.Get(k)
if err != nil {
return nil, errors.Wrap(err, "Localstore Get failed on blobBucket.Get")
}
if err = o.Unmarshal(v); err != nil {
return nil, errors.Wrap(err, "Localstore Get failed on Object.Unmarshal")
}
return o, nil
}

20
lib/localstore/has.go Normal file
View file

@ -0,0 +1,20 @@
package localstore
import (
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/pkg/errors"
)
func (l *localstore) Has(key refs.Address) (bool, error) {
var (
err error
k []byte
)
k, err = key.Hash()
if err != nil {
return false, errors.Wrap(err, "localstore.Has failed on key.Marshal")
}
return l.metaBucket.Has(k) && l.blobBucket.Has(k), nil
}

102
lib/localstore/interface.go Normal file
View file

@ -0,0 +1,102 @@
package localstore
import (
"context"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/nspcc-dev/neofs-node/lib/metrics"
"github.com/pkg/errors"
"go.uber.org/zap"
)
type (
// Localstore is an interface of local object storage.
Localstore interface {
Put(context.Context, *Object) error
Get(Address) (*Object, error)
Del(Address) error
Meta(Address) (*ObjectMeta, error)
Iterator
Has(Address) (bool, error)
ObjectsCount() (uint64, error)
object.PositionReader
Size() int64
}
// MetaHandler is a function that handles ObjectMeta.
MetaHandler func(*ObjectMeta) bool
// Iterator is an interface of the iterator over local object storage.
Iterator interface {
Iterate(FilterPipeline, MetaHandler) error
}
// ListItem is an ObjectMeta wrapper.
ListItem struct {
ObjectMeta
}
// Params groups the parameters of
// local object storage constructor.
Params struct {
BlobBucket core.Bucket
MetaBucket core.Bucket
Logger *zap.Logger
Collector metrics.Collector
}
localstore struct {
metaBucket core.Bucket
blobBucket core.Bucket
log *zap.Logger
col metrics.Collector
}
)
// ErrOutOfRange is returned when requested object payload range is
// out of object payload bounds.
var ErrOutOfRange = errors.New("range is out of payload bounds")
// ErrEmptyMetaHandler is returned by functions that expect
// a non-nil MetaHandler, but received nil.
var ErrEmptyMetaHandler = errors.New("meta handler is nil")
var errNilLogger = errors.New("logger is nil")
var errNilCollector = errors.New("metrics collector is nil")
// New is a local object storage constructor.
func New(p Params) (Localstore, error) {
switch {
case p.MetaBucket == nil:
return nil, errors.Errorf("%s bucket is nil", core.MetaStore)
case p.BlobBucket == nil:
return nil, errors.Errorf("%s bucket is nil", core.BlobStore)
case p.Logger == nil:
return nil, errNilLogger
case p.Collector == nil:
return nil, errNilCollector
}
return &localstore{
metaBucket: p.MetaBucket,
blobBucket: p.BlobBucket,
log: p.Logger,
col: p.Collector,
}, nil
}
func (l localstore) Size() int64 { return l.blobBucket.Size() }
// TODO: implement less costly method of counting.
func (l localstore) ObjectsCount() (uint64, error) {
items, err := l.metaBucket.List()
if err != nil {
return 0, err
}
return uint64(len(items)), nil
}

41
lib/localstore/list.go Normal file
View file

@ -0,0 +1,41 @@
package localstore
import (
"context"
"go.uber.org/zap"
)
func (l *localstore) Iterate(filter FilterPipeline, handler MetaHandler) error {
if handler == nil {
return ErrEmptyMetaHandler
} else if filter == nil {
filter = NewFilter(&FilterParams{
Name: "SKIPPING_FILTER",
FilterFunc: SkippingFilterFunc,
})
}
return l.metaBucket.Iterate(func(_, v []byte) bool {
meta := new(ObjectMeta)
if err := meta.Unmarshal(v); err != nil {
l.log.Error("unmarshal meta bucket item failure", zap.Error(err))
} else if filter.Pass(context.TODO(), meta).Code() == CodePass {
return !handler(meta)
}
return true
})
}
// ListItems iterates over Iterator with FilterPipeline and returns all passed items.
func ListItems(it Iterator, f FilterPipeline) ([]ListItem, error) {
res := make([]ListItem, 0)
err := it.Iterate(f, func(meta *ObjectMeta) (stop bool) {
res = append(res, ListItem{
ObjectMeta: *meta,
})
return
})
return res, err
}

View file

@ -0,0 +1,462 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: lib/localstore/localstore.proto
package localstore
import (
fmt "fmt"
_ "github.com/gogo/protobuf/gogoproto"
proto "github.com/golang/protobuf/proto"
object "github.com/nspcc-dev/neofs-api-go/object"
io "io"
math "math"
math_bits "math/bits"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
type ObjectMeta struct {
Object *object.Object `protobuf:"bytes,1,opt,name=Object,proto3" json:"Object,omitempty"`
PayloadHash Hash `protobuf:"bytes,2,opt,name=PayloadHash,proto3,customtype=Hash" json:"PayloadHash"`
PayloadSize uint64 `protobuf:"varint,3,opt,name=PayloadSize,proto3" json:"PayloadSize,omitempty"`
StoreEpoch uint64 `protobuf:"varint,4,opt,name=StoreEpoch,proto3" json:"StoreEpoch,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *ObjectMeta) Reset() { *m = ObjectMeta{} }
func (m *ObjectMeta) String() string { return proto.CompactTextString(m) }
func (*ObjectMeta) ProtoMessage() {}
func (*ObjectMeta) Descriptor() ([]byte, []int) {
return fileDescriptor_3236d71280f5b180, []int{0}
}
func (m *ObjectMeta) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *ObjectMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_ObjectMeta.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *ObjectMeta) XXX_Merge(src proto.Message) {
xxx_messageInfo_ObjectMeta.Merge(m, src)
}
func (m *ObjectMeta) XXX_Size() int {
return m.Size()
}
func (m *ObjectMeta) XXX_DiscardUnknown() {
xxx_messageInfo_ObjectMeta.DiscardUnknown(m)
}
var xxx_messageInfo_ObjectMeta proto.InternalMessageInfo
func (m *ObjectMeta) GetObject() *object.Object {
if m != nil {
return m.Object
}
return nil
}
func (m *ObjectMeta) GetPayloadSize() uint64 {
if m != nil {
return m.PayloadSize
}
return 0
}
func (m *ObjectMeta) GetStoreEpoch() uint64 {
if m != nil {
return m.StoreEpoch
}
return 0
}
func init() {
proto.RegisterType((*ObjectMeta)(nil), "localstore.ObjectMeta")
}
func init() { proto.RegisterFile("lib/localstore/localstore.proto", fileDescriptor_3236d71280f5b180) }
var fileDescriptor_3236d71280f5b180 = []byte{
// 257 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0xcf, 0xc9, 0x4c, 0xd2,
0xcf, 0xc9, 0x4f, 0x4e, 0xcc, 0x29, 0x2e, 0xc9, 0x2f, 0x4a, 0x45, 0x62, 0xea, 0x15, 0x14, 0xe5,
0x97, 0xe4, 0x0b, 0x71, 0x21, 0x44, 0xa4, 0x84, 0xf2, 0x93, 0xb2, 0x52, 0x93, 0x4b, 0xf4, 0x4b,
0x2a, 0x0b, 0x52, 0x8b, 0x21, 0xf2, 0x52, 0xba, 0xe9, 0x99, 0x25, 0x19, 0xa5, 0x49, 0x7a, 0xc9,
0xf9, 0xb9, 0xfa, 0xe9, 0xf9, 0xe9, 0xf9, 0xfa, 0x60, 0xe1, 0xa4, 0xd2, 0x34, 0x30, 0x0f, 0xcc,
0x01, 0xb3, 0x20, 0xca, 0x95, 0x96, 0x31, 0x72, 0x71, 0xf9, 0x83, 0x4d, 0xf1, 0x4d, 0x2d, 0x49,
0x14, 0x52, 0xe3, 0x62, 0x83, 0xf0, 0x24, 0x18, 0x15, 0x18, 0x35, 0xb8, 0x8d, 0xf8, 0xf4, 0x20,
0x56, 0xe8, 0x41, 0x44, 0x83, 0xa0, 0xb2, 0x42, 0x7a, 0x5c, 0xdc, 0x01, 0x89, 0x95, 0x39, 0xf9,
0x89, 0x29, 0x1e, 0x89, 0xc5, 0x19, 0x12, 0x4c, 0x0a, 0x8c, 0x1a, 0x3c, 0x4e, 0x3c, 0x27, 0xee,
0xc9, 0x33, 0xdc, 0xba, 0x27, 0xcf, 0x02, 0x12, 0x0b, 0x42, 0x56, 0x20, 0xa4, 0x00, 0x57, 0x1f,
0x9c, 0x59, 0x95, 0x2a, 0xc1, 0xac, 0xc0, 0xa8, 0xc1, 0x12, 0x84, 0x2c, 0x24, 0x24, 0xc7, 0xc5,
0x15, 0x0c, 0xf2, 0x94, 0x6b, 0x41, 0x7e, 0x72, 0x86, 0x04, 0x0b, 0x58, 0x01, 0x92, 0x88, 0x93,
0xc3, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe3, 0xb1,
0x1c, 0x43, 0x94, 0x1e, 0x92, 0x4f, 0xf3, 0x8a, 0x0b, 0x92, 0x93, 0x75, 0x53, 0x52, 0xcb, 0xf4,
0xf3, 0x52, 0xf3, 0xd3, 0x8a, 0x75, 0xf3, 0xf2, 0x53, 0x52, 0xf5, 0x51, 0x83, 0x32, 0x89, 0x0d,
0xec, 0x63, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xc8, 0x20, 0xb3, 0xa4, 0x63, 0x01, 0x00,
0x00,
}
func (m *ObjectMeta) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *ObjectMeta) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *ObjectMeta) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.XXX_unrecognized != nil {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.StoreEpoch != 0 {
i = encodeVarintLocalstore(dAtA, i, uint64(m.StoreEpoch))
i--
dAtA[i] = 0x20
}
if m.PayloadSize != 0 {
i = encodeVarintLocalstore(dAtA, i, uint64(m.PayloadSize))
i--
dAtA[i] = 0x18
}
{
size := m.PayloadHash.Size()
i -= size
if _, err := m.PayloadHash.MarshalTo(dAtA[i:]); err != nil {
return 0, err
}
i = encodeVarintLocalstore(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x12
if m.Object != nil {
{
size, err := m.Object.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintLocalstore(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func encodeVarintLocalstore(dAtA []byte, offset int, v uint64) int {
offset -= sovLocalstore(v)
base := offset
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
dAtA[offset] = uint8(v)
return base
}
func (m *ObjectMeta) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if m.Object != nil {
l = m.Object.Size()
n += 1 + l + sovLocalstore(uint64(l))
}
l = m.PayloadHash.Size()
n += 1 + l + sovLocalstore(uint64(l))
if m.PayloadSize != 0 {
n += 1 + sovLocalstore(uint64(m.PayloadSize))
}
if m.StoreEpoch != 0 {
n += 1 + sovLocalstore(uint64(m.StoreEpoch))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func sovLocalstore(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
func sozLocalstore(x uint64) (n int) {
return sovLocalstore(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *ObjectMeta) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLocalstore
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: ObjectMeta: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: ObjectMeta: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Object", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLocalstore
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthLocalstore
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthLocalstore
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Object == nil {
m.Object = &object.Object{}
}
if err := m.Object.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field PayloadHash", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLocalstore
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthLocalstore
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthLocalstore
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if err := m.PayloadHash.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field PayloadSize", wireType)
}
m.PayloadSize = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLocalstore
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.PayloadSize |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field StoreEpoch", wireType)
}
m.StoreEpoch = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLocalstore
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.StoreEpoch |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipLocalstore(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthLocalstore
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthLocalstore
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipLocalstore(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
depth := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLocalstore
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLocalstore
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if dAtA[iNdEx-1] < 0x80 {
break
}
}
case 1:
iNdEx += 8
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLocalstore
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if length < 0 {
return 0, ErrInvalidLengthLocalstore
}
iNdEx += length
case 3:
depth++
case 4:
if depth == 0 {
return 0, ErrUnexpectedEndOfGroupLocalstore
}
depth--
case 5:
iNdEx += 4
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
if iNdEx < 0 {
return 0, ErrInvalidLengthLocalstore
}
if depth == 0 {
return iNdEx, nil
}
}
return 0, io.ErrUnexpectedEOF
}
var (
ErrInvalidLengthLocalstore = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowLocalstore = fmt.Errorf("proto: integer overflow")
ErrUnexpectedEndOfGroupLocalstore = fmt.Errorf("proto: unexpected end of group")
)

View file

@ -0,0 +1,14 @@
syntax = "proto3";
option go_package = "github.com/nspcc-dev/neofs-node/lib/localstore";
package localstore;
import "object/types.proto";
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
message ObjectMeta {
object.Object Object = 1;
bytes PayloadHash = 2 [(gogoproto.nullable) = false, (gogoproto.customtype) = "Hash"];
uint64 PayloadSize = 3;
uint64 StoreEpoch = 4;
}

View file

@ -0,0 +1,501 @@
package localstore
import (
"context"
"sync"
"testing"
"github.com/google/uuid"
"github.com/nspcc-dev/neofs-api-go/container"
"github.com/nspcc-dev/neofs-api-go/hash"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-node/lib/meta"
"github.com/nspcc-dev/neofs-node/lib/metrics"
"github.com/nspcc-dev/neofs-node/lib/test"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
type (
testBucket struct {
sync.RWMutex
items map[string][]byte
}
fakeCollector struct {
sync.Mutex
items map[refs.CID]uint64
}
)
func (f *fakeCollector) Start(_ context.Context) { panic("implement me") }
func (f *fakeCollector) UpdateSpaceUsage() { panic("implement me") }
func (f *fakeCollector) SetIterator(_ meta.Iterator) { panic("implement me") }
func (f *fakeCollector) SetCounter(counter metrics.ObjectCounter) { panic("implement me") }
func (f *fakeCollector) UpdateContainer(cid refs.CID, size uint64, op metrics.SpaceOp) {
f.Lock()
defer f.Unlock()
switch op {
case metrics.AddSpace:
f.items[cid] += size
case metrics.RemSpace:
if val, ok := f.items[cid]; !ok || val < size {
return
}
f.items[cid] -= size
default:
return
}
}
func newCollector() metrics.Collector {
return &fakeCollector{
items: make(map[refs.CID]uint64),
}
}
func newTestBucket() *testBucket {
return &testBucket{
items: make(map[string][]byte),
}
}
//
// func (t *testBucket) Get(key []byte) ([]byte, error) {
// t.Lock()
// defer t.Unlock()
//
// val, ok := t.items[base58.Encode(key)]
// if !ok {
// return nil, errors.New("item not found")
// }
//
// return val, nil
// }
//
// func (t *testBucket) Set(key, value []byte) error {
// t.Lock()
// defer t.Unlock()
//
// t.items[base58.Encode(key)] = value
//
// return nil
// }
//
// func (t *testBucket) Del(key []byte) error {
// t.RLock()
// defer t.RUnlock()
//
// delete(t.items, base58.Encode(key))
//
// return nil
// }
//
// func (t *testBucket) Has(key []byte) bool {
// panic("implement me")
// }
//
// func (t *testBucket) Size() int64 {
// panic("implement me")
// }
//
// func (t *testBucket) List() ([]core.BucketItem, error) {
// t.Lock()
// defer t.Unlock()
//
// res := make([]core.BucketItem, 0)
//
// for k, v := range t.items {
// sk, err := base58.Decode(k)
// if err != nil {
// return nil, err
// }
//
// res = append(res, core.BucketItem{
// Key: sk,
// Val: v,
// })
// }
//
// return res, nil
// }
//
// func (t *testBucket) Filter(core.FilterHandler) ([]core.BucketItem, error) {
// panic("implement me")
// }
//
// func (t *testBucket) Close() error {
// panic("implement me")
// }
//
// func (t *testBucket) PRead(key []byte, rng object.Range) ([]byte, error) {
// panic("implement me")
// }
func testObject(t *testing.T) *Object {
var (
uid refs.UUID
cid CID
)
t.Run("Prepare object", func(t *testing.T) {
cnr, err := container.NewTestContainer()
require.NoError(t, err)
cid, err = cnr.ID()
require.NoError(t, err)
id, err := uuid.NewRandom()
uid = refs.UUID(id)
require.NoError(t, err)
})
obj := &Object{
SystemHeader: object.SystemHeader{
Version: 1,
ID: uid,
CID: cid,
OwnerID: refs.OwnerID([refs.OwnerIDSize]byte{}), // TODO: avoid hardcode
},
Headers: []Header{
{
Value: &object.Header_UserHeader{
UserHeader: &object.UserHeader{
Key: "Profession",
Value: "Developer",
},
},
},
{
Value: &object.Header_UserHeader{
UserHeader: &object.UserHeader{
Key: "Language",
Value: "GO",
},
},
},
},
}
return obj
}
func newLocalstore(t *testing.T) Localstore {
ls, err := New(Params{
BlobBucket: test.Bucket(),
MetaBucket: test.Bucket(),
Logger: zap.L(),
Collector: newCollector(),
})
require.NoError(t, err)
return ls
}
func TestNew(t *testing.T) {
t.Run("New localstore", func(t *testing.T) {
var err error
_, err = New(Params{})
require.Error(t, err)
_, err = New(Params{
BlobBucket: test.Bucket(),
MetaBucket: test.Bucket(),
Logger: zap.L(),
Collector: newCollector(),
})
require.NoError(t, err)
})
}
func TestLocalstore_Del(t *testing.T) {
t.Run("Del method", func(t *testing.T) {
var (
err error
ls Localstore
obj *Object
)
ls = newLocalstore(t)
obj = testObject(t)
obj.SetPayload([]byte("Hello, world"))
k := *obj.Address()
store, ok := ls.(*localstore)
require.True(t, ok)
require.NotNil(t, store)
metric, ok := store.col.(*fakeCollector)
require.True(t, ok)
require.NotNil(t, metric)
err = ls.Put(context.Background(), obj)
require.NoError(t, err)
require.NotEmpty(t, obj.Payload)
require.Contains(t, metric.items, obj.SystemHeader.CID)
require.Equal(t, obj.SystemHeader.PayloadLength, metric.items[obj.SystemHeader.CID])
err = ls.Del(k)
require.NoError(t, err)
require.Contains(t, metric.items, obj.SystemHeader.CID)
require.Equal(t, uint64(0), metric.items[obj.SystemHeader.CID])
_, err = ls.Get(k)
require.Error(t, err)
})
}
func TestLocalstore_Get(t *testing.T) {
t.Run("Get method (default)", func(t *testing.T) {
var (
err error
ls Localstore
obj *Object
)
ls = newLocalstore(t)
obj = testObject(t)
err = ls.Put(context.Background(), obj)
require.NoError(t, err)
k := *obj.Address()
o, err := ls.Get(k)
require.NoError(t, err)
require.Equal(t, obj, o)
})
}
func TestLocalstore_Put(t *testing.T) {
t.Run("Put method", func(t *testing.T) {
var (
err error
ls Localstore
obj *Object
)
ls = newLocalstore(t)
store, ok := ls.(*localstore)
require.True(t, ok)
require.NotNil(t, store)
metric, ok := store.col.(*fakeCollector)
require.True(t, ok)
require.NotNil(t, metric)
obj = testObject(t)
err = ls.Put(context.Background(), obj)
require.NoError(t, err)
require.Contains(t, metric.items, obj.SystemHeader.CID)
require.Equal(t, obj.SystemHeader.PayloadLength, metric.items[obj.SystemHeader.CID])
o, err := ls.Get(*obj.Address())
require.NoError(t, err)
require.Equal(t, obj, o)
})
}
func TestLocalstore_List(t *testing.T) {
t.Run("List method (no filters)", func(t *testing.T) {
var (
err error
ls Localstore
objCount = 10
objs = make([]Object, objCount)
)
for i := range objs {
objs[i] = *testObject(t)
}
ls = newLocalstore(t)
for i := range objs {
err = ls.Put(context.Background(), &objs[i])
require.NoError(t, err)
}
items, err := ListItems(ls, nil)
require.NoError(t, err)
require.Len(t, items, objCount)
for i := range items {
require.Contains(t, objs, *items[i].Object)
}
})
t.Run("List method ('bad' filter)", func(t *testing.T) {
var (
err error
ls Localstore
objCount = 10
objs = make([]Object, objCount)
)
for i := range objs {
objs[i] = *testObject(t)
}
ls = newLocalstore(t)
for i := range objs {
err = ls.Put(context.Background(), &objs[i])
require.NoError(t, err)
}
items, err := ListItems(ls, NewFilter(&FilterParams{
FilterFunc: ContainerFilterFunc([]CID{}),
}))
require.NoError(t, err)
require.Len(t, items, 0)
})
t.Run("List method (filter by cid)", func(t *testing.T) {
var (
err error
ls Localstore
objCount = 10
objs = make([]Object, objCount)
)
for i := range objs {
objs[i] = *testObject(t)
}
ls = newLocalstore(t)
for i := range objs {
err = ls.Put(context.Background(), &objs[i])
require.NoError(t, err)
}
cidVals := []CID{objs[0].SystemHeader.CID}
items, err := ListItems(ls, NewFilter(&FilterParams{
FilterFunc: ContainerFilterFunc(cidVals),
}))
require.NoError(t, err)
require.Len(t, items, 1)
for i := range items {
require.Contains(t, objs, *items[i].Object)
}
})
t.Run("Filter stored earlier", func(t *testing.T) {
var (
err error
ls Localstore
objCount = 10
objs = make([]Object, objCount)
epoch uint64 = 100
list []ListItem
)
for i := range objs {
objs[i] = *testObject(t)
}
ls = newLocalstore(t)
ctx := context.WithValue(context.Background(), StoreEpochValue, epoch)
for i := range objs {
err = ls.Put(ctx, &objs[i])
require.NoError(t, err)
}
list, err = ListItems(ls, NewFilter(&FilterParams{
FilterFunc: StoredEarlierThanFilterFunc(epoch - 1),
}))
require.NoError(t, err)
require.Empty(t, list)
list, err = ListItems(ls, NewFilter(&FilterParams{
FilterFunc: StoredEarlierThanFilterFunc(epoch),
}))
require.NoError(t, err)
require.Empty(t, list)
list, err = ListItems(ls, NewFilter(&FilterParams{
FilterFunc: StoredEarlierThanFilterFunc(epoch + 1),
}))
require.NoError(t, err)
require.Len(t, list, objCount)
})
t.Run("Filter with complex filter", func(t *testing.T) {
var (
err error
ls Localstore
objCount = 10
objs = make([]Object, objCount)
)
for i := range objs {
objs[i] = *testObject(t)
}
ls = newLocalstore(t)
for i := range objs {
err = ls.Put(context.Background(), &objs[i])
require.NoError(t, err)
}
cidVals := []CID{objs[0].SystemHeader.CID}
mainF, err := AllPassIncludingFilter("TEST_FILTER", &FilterParams{
Name: "CID_LIST",
FilterFunc: ContainerFilterFunc(cidVals),
})
items, err := ListItems(ls, mainF)
require.NoError(t, err)
require.Len(t, items, 1)
})
t.Run("Meta info", func(t *testing.T) {
var (
err error
ls Localstore
objCount = 10
objs = make([]Object, objCount)
epoch uint64 = 100
)
for i := range objs {
objs[i] = *testObject(t)
}
ls = newLocalstore(t)
ctx := context.WithValue(context.Background(), StoreEpochValue, epoch)
for i := range objs {
err = ls.Put(ctx, &objs[i])
require.NoError(t, err)
meta, err := ls.Meta(*objs[i].Address())
require.NoError(t, err)
noPayload := objs[i]
noPayload.Payload = nil
require.Equal(t, *meta.Object, noPayload)
require.Equal(t, meta.PayloadHash, hash.Sum(objs[i].Payload))
require.Equal(t, meta.PayloadSize, uint64(len(objs[i].Payload)))
require.Equal(t, epoch, meta.StoreEpoch)
}
})
}

52
lib/localstore/meta.go Normal file
View file

@ -0,0 +1,52 @@
package localstore
import (
"context"
"github.com/nspcc-dev/neofs-api-go/hash"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/pkg/errors"
)
// StoreEpochValue is a context key of object storing epoch number.
const StoreEpochValue = "store epoch"
func (l *localstore) Meta(key refs.Address) (*ObjectMeta, error) {
var (
err error
meta ObjectMeta
k, v []byte
)
k, err = key.Hash()
if err != nil {
return nil, errors.Wrap(err, "Localstore Meta failed on key.Marshal")
}
v, err = l.metaBucket.Get(k)
if err != nil {
return nil, errors.Wrap(err, "Localstore Meta failed on metaBucket.Get")
}
if err := meta.Unmarshal(v); err != nil {
return nil, errors.Wrap(err, "Localstore Metafailed on ObjectMeta.Unmarshal")
}
return &meta, nil
}
func metaFromObject(ctx context.Context, obj *Object) *ObjectMeta {
meta := new(ObjectMeta)
o := *obj
meta.Object = &o
meta.Object.Payload = nil
meta.PayloadSize = uint64(len(obj.Payload))
meta.PayloadHash = hash.Sum(obj.Payload)
storeEpoch, ok := ctx.Value(StoreEpochValue).(uint64)
if ok {
meta.StoreEpoch = storeEpoch
}
return meta
}

47
lib/localstore/put.go Normal file
View file

@ -0,0 +1,47 @@
package localstore
import (
"context"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-node/lib/metrics"
"github.com/pkg/errors"
)
func (l *localstore) Put(ctx context.Context, obj *Object) error {
var (
oa refs.Address
k, v []byte
err error
)
oa = *obj.Address()
k, err = oa.Hash()
if err != nil {
return errors.Wrap(err, "Localstore Put failed on StorageKey.marshal")
}
if v, err = obj.Marshal(); err != nil {
return errors.Wrap(err, "Localstore Put failed on blobValue")
}
if err = l.blobBucket.Set(k, v); err != nil {
return errors.Wrap(err, "Localstore Put failed on BlobBucket.Set")
}
if v, err = metaFromObject(ctx, obj).Marshal(); err != nil {
return errors.Wrap(err, "Localstore Put failed on metaValue")
}
if err = l.metaBucket.Set(k, v); err != nil {
return errors.Wrap(err, "Localstore Put failed on MetaBucket.Set")
}
l.col.UpdateContainer(
obj.SystemHeader.CID,
obj.SystemHeader.PayloadLength,
metrics.AddSpace)
return nil
}

36
lib/localstore/range.go Normal file
View file

@ -0,0 +1,36 @@
package localstore
import (
"context"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/pkg/errors"
)
func (l *localstore) PRead(ctx context.Context, key Address, rng object.Range) ([]byte, error) {
var (
err error
k, v []byte
obj Object
)
k, err = key.Hash()
if err != nil {
return nil, errors.Wrap(err, "Localstore Get failed on key.Marshal")
}
v, err = l.blobBucket.Get(k)
if err != nil {
return nil, errors.Wrap(err, "Localstore Get failed on blobBucket.Get")
}
if err := obj.Unmarshal(v); err != nil {
return nil, errors.Wrap(err, "Localstore Get failed on object.Unmarshal")
}
if rng.Offset+rng.Length > uint64(len(obj.Payload)) {
return nil, ErrOutOfRange
}
return obj.Payload[rng.Offset : rng.Offset+rng.Length], nil
}

15
lib/meta/iterator.go Normal file
View file

@ -0,0 +1,15 @@
package meta
import (
"github.com/nspcc-dev/neofs-api-go/object"
)
type (
// Iterator is an interface of the iterator over object storage.
Iterator interface {
Iterate(IterateFunc) error
}
// IterateFunc is a function that checks whether an object matches a specific criterion.
IterateFunc func(*object.Object) error
)

33
lib/metrics/meta.go Normal file
View file

@ -0,0 +1,33 @@
package metrics
import (
"sync"
"github.com/nspcc-dev/neofs-node/lib/meta"
)
type metaWrapper struct {
sync.Mutex
iter meta.Iterator
}
func newMetaWrapper() *metaWrapper {
return &metaWrapper{}
}
func (m *metaWrapper) changeIter(iter meta.Iterator) {
m.Lock()
m.iter = iter
m.Unlock()
}
func (m *metaWrapper) Iterate(h meta.IterateFunc) error {
m.Lock()
defer m.Unlock()
if m.iter == nil {
return errEmptyMetaStore
}
return m.iter.Iterate(h)
}

175
lib/metrics/metrics.go Normal file
View file

@ -0,0 +1,175 @@
package metrics
import (
"context"
"sync"
"time"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/nspcc-dev/neofs-node/lib/meta"
"go.uber.org/zap"
)
type (
// Collector is an interface of the metrics collector.
Collector interface {
Start(ctx context.Context)
UpdateSpaceUsage()
SetCounter(ObjectCounter)
SetIterator(iter meta.Iterator)
UpdateContainer(cid refs.CID, size uint64, op SpaceOp)
}
collector struct {
log *zap.Logger
interval time.Duration
counter *counterWrapper
sizes *syncStore
metas *metaWrapper
updateSpaceSize func()
updateObjectCount func()
}
// Params groups the parameters of metrics collector's constructor.
Params struct {
Options []string
Logger *zap.Logger
Interval time.Duration
MetricsStore core.Bucket
}
// ObjectCounter is an interface of object number storage.
ObjectCounter interface {
ObjectsCount() (uint64, error)
}
// CounterSetter is an interface of ObjectCounter container.
CounterSetter interface {
SetCounter(ObjectCounter)
}
counterWrapper struct {
sync.Mutex
counter ObjectCounter
}
)
const (
errEmptyCounter = internal.Error("empty object counter")
errEmptyLogger = internal.Error("empty logger")
errEmptyMetaStore = internal.Error("empty meta store")
errEmptyMetricsStore = internal.Error("empty metrics store")
defaultMetricsInterval = 5 * time.Second
)
// New constructs metrics collector and returns Collector interface.
func New(p Params) (Collector, error) {
switch {
case p.Logger == nil:
return nil, errEmptyLogger
case p.MetricsStore == nil:
return nil, errEmptyMetricsStore
}
if p.Interval <= 0 {
p.Interval = defaultMetricsInterval
}
metas := newMetaWrapper()
sizes := newSyncStore(p.Logger, p.MetricsStore)
sizes.Load()
return &collector{
log: p.Logger,
interval: p.Interval,
counter: new(counterWrapper),
metas: metas,
sizes: sizes,
updateSpaceSize: spaceUpdater(sizes),
updateObjectCount: metricsUpdater(p.Options),
}, nil
}
func (c *counterWrapper) SetCounter(counter ObjectCounter) {
c.Lock()
defer c.Unlock()
c.counter = counter
}
func (c *counterWrapper) ObjectsCount() (uint64, error) {
c.Lock()
defer c.Unlock()
if c.counter == nil {
return 0, errEmptyCounter
}
return c.counter.ObjectsCount()
}
func (c *collector) SetCounter(counter ObjectCounter) {
c.counter.SetCounter(counter)
}
func (c *collector) SetIterator(iter meta.Iterator) {
c.metas.changeIter(iter)
}
func (c *collector) UpdateContainer(cid refs.CID, size uint64, op SpaceOp) {
c.sizes.Update(cid, size, op)
c.updateSpaceSize()
}
func (c *collector) UpdateSpaceUsage() {
sizes := make(map[refs.CID]uint64)
err := c.metas.Iterate(func(obj *object.Object) error {
if !obj.IsTombstone() {
cid := obj.SystemHeader.CID
sizes[cid] += obj.SystemHeader.PayloadLength
}
return nil
})
if err != nil {
c.log.Error("could not update space metrics", zap.Error(err))
}
c.sizes.Reset(sizes)
c.updateSpaceSize()
}
func (c *collector) Start(ctx context.Context) {
t := time.NewTicker(c.interval)
loop:
for {
select {
case <-ctx.Done():
c.log.Warn("stop collecting metrics", zap.Error(ctx.Err()))
break loop
case <-t.C:
count, err := c.counter.ObjectsCount()
if err != nil {
c.log.Warn("get object count failure", zap.Error(err))
continue loop
}
counter.Store(float64(count))
c.updateObjectCount()
}
}
t.Stop()
}

275
lib/metrics/metrics_test.go Normal file
View file

@ -0,0 +1,275 @@
package metrics
import (
"context"
"encoding/binary"
"sync"
"testing"
"time"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-node/lib/meta"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
type (
fakeCounter int
fakeIterator string
fakeMetaStore []*object.Object
)
var (
_ ObjectCounter = (*fakeCounter)(nil)
_ meta.Iterator = (*fakeIterator)(nil)
)
func (f fakeCounter) ObjectsCount() (uint64, error) {
return uint64(f), nil
}
func (f fakeIterator) Iterate(_ meta.IterateFunc) error {
if f == "" {
return nil
}
return errors.New(string(f))
}
func (f fakeMetaStore) Iterate(cb meta.IterateFunc) error {
if cb == nil {
return nil
}
for i := range f {
if err := cb(f[i]); err != nil {
return err
}
}
return nil
}
func TestCollector(t *testing.T) {
buck := &fakeBucket{items: make(map[uint64]int)}
t.Run("check errors", func(t *testing.T) {
t.Run("empty logger", func(t *testing.T) {
svc, err := New(Params{MetricsStore: buck})
require.Nil(t, svc)
require.EqualError(t, err, errEmptyLogger.Error())
})
t.Run("empty metrics store", func(t *testing.T) {
svc, err := New(Params{Logger: zap.L()})
require.Nil(t, svc)
require.EqualError(t, err, errEmptyMetricsStore.Error())
})
})
svc, err := New(Params{
Logger: zap.L(),
MetricsStore: buck,
Options: []string{
"/Location:Europe/Country:Russia/City:Moscow",
"/Some:Another/Key:Value",
},
})
require.NoError(t, err)
require.NotNil(t, svc)
coll, ok := svc.(*collector)
require.True(t, ok)
require.NotNil(t, coll)
t.Run("check start", func(t *testing.T) {
coll.interval = time.Second
t.Run("stop by context", func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
wg := new(sync.WaitGroup)
wg.Add(1)
counter.Store(-1)
go func() {
svc.Start(ctx)
wg.Done()
}()
cancel()
wg.Wait()
require.Equal(t, float64(-1), counter.Load())
})
t.Run("should fail on empty counter", func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
wg := new(sync.WaitGroup)
wg.Add(1)
counter.Store(0)
go func() {
svc.Start(ctx)
wg.Done()
}()
time.Sleep(2 * time.Second)
cancel()
wg.Wait()
require.Equal(t, float64(0), counter.Load())
})
t.Run("should success on fakeCounter", func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
wg := new(sync.WaitGroup)
wg.Add(1)
coll.SetCounter(fakeCounter(8))
counter.Store(0)
go func() {
svc.Start(ctx)
wg.Done()
}()
time.Sleep(2 * time.Second)
cancel()
wg.Wait()
require.Equal(t, float64(8), counter.Load())
})
})
t.Run("iterator", func(t *testing.T) {
{
coll.SetIterator(nil)
require.Nil(t, coll.metas.iter)
require.EqualError(t, coll.metas.Iterate(nil), errEmptyMetaStore.Error())
}
{
iter := fakeIterator("")
coll.SetIterator(iter)
require.Equal(t, iter, coll.metas.iter)
require.NoError(t, coll.metas.Iterate(nil))
}
{
iter := fakeIterator("test")
coll.SetIterator(iter)
require.Equal(t, iter, coll.metas.iter)
require.EqualError(t, coll.metas.Iterate(nil), string(iter))
}
})
t.Run("add-rem space", func(t *testing.T) {
cid := refs.CID{1, 2, 3, 4, 5}
buf := make([]byte, 8)
key := keyFromBytes(cid.Bytes())
zero := make([]byte, 8)
size := uint64(100)
binary.BigEndian.PutUint64(buf, size)
{
coll.UpdateContainer(cid, size, AddSpace)
require.Len(t, coll.sizes.items, 1)
require.Len(t, buck.items, 1)
require.Contains(t, buck.items, key)
require.Contains(t, buck.kv, fakeKV{key: cid.Bytes(), val: buf})
}
{
coll.UpdateContainer(cid, size, RemSpace)
require.Len(t, coll.sizes.items, 1)
require.Len(t, buck.items, 1)
require.Contains(t, buck.items, key)
require.Contains(t, buck.kv, fakeKV{key: cid.Bytes(), val: zero})
}
{
coll.UpdateContainer(cid, size, RemSpace)
require.Len(t, coll.sizes.items, 1)
require.Len(t, buck.items, 1)
require.Contains(t, buck.kv, fakeKV{key: cid.Bytes(), val: zero})
}
})
t.Run("add-rem multi thread", func(t *testing.T) {
wg := new(sync.WaitGroup)
wg.Add(10)
size := uint64(100)
zero := make([]byte, 8)
// reset
coll.UpdateSpaceUsage()
for i := 0; i < 10; i++ {
cid := refs.CID{1, 2, 3, 4, byte(i)}
coll.UpdateContainer(cid, size, AddSpace)
go func() {
coll.UpdateContainer(cid, size, RemSpace)
wg.Done()
}()
}
wg.Wait()
require.Len(t, coll.sizes.items, 10)
require.Len(t, buck.items, 10)
for i := 0; i < 10; i++ {
cid := refs.CID{1, 2, 3, 4, byte(i)}
require.Contains(t, buck.kv, fakeKV{key: cid.Bytes(), val: zero})
}
})
t.Run("reset buckets", func(t *testing.T) {
coll.UpdateSpaceUsage()
require.Len(t, coll.sizes.items, 0)
require.Len(t, buck.items, 0)
})
t.Run("reset from metaStore", func(t *testing.T) {
cid := refs.CID{1, 2, 3, 4, 5}
buf := make([]byte, 8)
key := keyFromBytes(cid.Bytes())
size := uint64(100)
binary.BigEndian.PutUint64(buf, size)
iter := fakeMetaStore{
{
SystemHeader: object.SystemHeader{
PayloadLength: size,
CID: cid,
},
},
{
Headers: []object.Header{
{
Value: &object.Header_Tombstone{Tombstone: &object.Tombstone{}},
},
},
},
}
coll.SetIterator(iter)
coll.UpdateSpaceUsage()
require.Len(t, coll.sizes.items, 1)
require.Len(t, buck.items, 1)
require.Contains(t, buck.items, key)
require.Contains(t, buck.kv, fakeKV{key: cid.Bytes(), val: buf})
})
}

Some files were not shown because too many files have changed in this diff Show more