[#128] localstorage: Implement primary object metabase

Implement bolt-based metabase that is going to be used in local object
storage. Implement Put/Get/Select methods.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/multiformats/go-multiaddr-net v0.1.2 // v0.1.1 => v0.1.2
 	github.com/multiformats/go-multihash v0.0.13 // indirect
 	github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78
-	github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201020152448-c8f46f7d9762
+	github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201028111149-ac38d13f04ff
 	github.com/nspcc-dev/neofs-crypto v0.3.0
 	github.com/nspcc-dev/tzhash v1.4.0
 	github.com/panjf2000/ants/v2 v2.3.0
@@ -25,12 +25,14 @@ require (
 	github.com/spf13/viper v1.7.0
 	github.com/stretchr/testify v1.6.1
 	github.com/valyala/fasthttp v1.9.0
-	go.etcd.io/bbolt v1.3.4
+	go.etcd.io/bbolt v1.3.5
 	go.uber.org/atomic v1.5.1
 	go.uber.org/multierr v1.4.0 // indirect
 	go.uber.org/zap v1.13.0
 	golang.org/x/crypto v0.0.0-20200117160349-530e935923ad // indirect
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
+	golang.org/x/net v0.0.0-20191105084925-a882066a44e0 // indirect
+	golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5 // indirect
 	golang.org/x/tools v0.0.0-20200123022218-593de606220b // indirect
 	google.golang.org/grpc v1.29.1
 	google.golang.org/protobuf v1.23.0

go.sum

@@ -71,6 +71,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -270,8 +271,8 @@ github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:
 github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=
 github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78 h1:stIa+nBXK8uDY/JZaxIZzAUfkzfaotVw2FbnHxO4aZI=
 github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=
-github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201020152448-c8f46f7d9762 h1:5pCgsBC8XQHX8U+ZBMsX7+87fw7nSMPBwHrewvKLXlk=
+github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201028111149-ac38d13f04ff h1:TBybhFCjTdSgpw0zGLw7ucjA3gTTLZDfp4D2trcSyT4=
-github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201020152448-c8f46f7d9762/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8=
+github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201028111149-ac38d13f04ff/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8=
 github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA=
 github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw=
 github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM=
@@ -390,6 +391,8 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
 go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -458,6 +461,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191105084925-a882066a44e0 h1:QPlSTtPE2k6PZPasQUbzuK3p9JbS+vMXYVto8g/yrsg=
+golang.org/x/net v0.0.0-20191105084925-a882066a44e0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
@@ -496,6 +501,8 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=
 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5 h1:iCaAy5bMeEvwANu3YnJfWwI0kWAGkEa2RXPdweI/ysk=
+golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=

pkg/local_object_storage/metabase/db.go

@@ -0,0 +1,25 @@
+package meta
+
+import (
+	"github.com/nspcc-dev/neofs-api-go/pkg/object"
+	"go.etcd.io/bbolt"
+)
+
+// DB represents local metabase of storage node.
+type DB struct {
+	boltDB *bbolt.DB
+
+	matchers map[object.SearchMatchType]func(string, string) bool
+}
+
+// NewDB creates, initializes and returns DB instance.
+func NewDB(boltDB *bbolt.DB) *DB {
+	return &DB{
+		boltDB: boltDB,
+		matchers: map[object.SearchMatchType]func(string, string) bool{
+			object.MatchStringEqual: func(s string, s2 string) bool {
+				return s == s2
+			},
+		},
+	}
+}

pkg/local_object_storage/metabase/db_test.go

@@ -0,0 +1,102 @@
+package meta
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"os"
+	"testing"
+
+	"github.com/nspcc-dev/neofs-api-go/pkg"
+	"github.com/nspcc-dev/neofs-api-go/pkg/container"
+	objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
+	"github.com/nspcc-dev/neofs-api-go/pkg/owner"
+	"github.com/nspcc-dev/neofs-node/pkg/core/object"
+	"github.com/nspcc-dev/neofs-node/pkg/util/test"
+	"github.com/stretchr/testify/require"
+	"go.etcd.io/bbolt"
+)
+
+func testSelect(t *testing.T, db *DB, fs objectSDK.SearchFilters, exp ...*objectSDK.Address) {
+	res, err := db.Select(fs)
+	require.NoError(t, err)
+	require.Len(t, res, len(exp))
+
+	for i := range exp {
+		require.Contains(t, res, exp[i])
+	}
+}
+
+func TestDB(t *testing.T) {
+	version := pkg.NewVersion()
+	version.SetMajor(2)
+	version.SetMinor(1)
+
+	cs := [sha256.Size]byte{}
+	rand.Read(cs[:])
+
+	cid := container.NewID()
+	cid.SetSHA256(cs)
+
+	w, err := owner.NEO3WalletFromPublicKey(&test.DecodeKey(-1).PublicKey)
+	require.NoError(t, err)
+
+	ownerID := owner.NewID()
+	ownerID.SetNeo3Wallet(w)
+
+	rand.Read(cs[:])
+	oid := objectSDK.NewID()
+	oid.SetSHA256(cs)
+
+	obj := object.NewRaw()
+	obj.SetID(oid)
+	obj.SetOwnerID(ownerID)
+	obj.SetContainerID(cid)
+	obj.SetVersion(version)
+
+	k, v := "key", "value"
+
+	a := objectSDK.NewAttribute()
+	a.SetKey(k)
+	a.SetValue(v)
+
+	obj.SetAttributes(a)
+
+	path := "test.db"
+
+	bdb, err := bbolt.Open(path, 0600, nil)
+	require.NoError(t, err)
+
+	defer func() {
+		bdb.Close()
+		os.Remove(path)
+	}()
+
+	db := NewDB(bdb)
+
+	o := obj.Object()
+
+	require.NoError(t, db.Put(o))
+
+	o2, err := db.Get(o.Address())
+	require.NoError(t, err)
+
+	require.Equal(t, o, o2)
+
+	fs := objectSDK.SearchFilters{}
+
+	// filter container ID
+	fs.AddObjectContainerIDFilter(objectSDK.MatchStringEqual, cid)
+	testSelect(t, db, fs, o.Address())
+
+	// filter owner ID
+	fs.AddObjectOwnerIDFilter(objectSDK.MatchStringEqual, ownerID)
+	testSelect(t, db, fs, o.Address())
+
+	// filter attribute
+	fs.AddFilter(k, v, objectSDK.MatchStringEqual)
+	testSelect(t, db, fs, o.Address())
+
+	// filter mismatch
+	fs.AddFilter(k, v+"1", objectSDK.MatchStringEqual)
+	testSelect(t, db, fs)
+}

pkg/local_object_storage/metabase/get.go

@@ -0,0 +1,38 @@
+package meta
+
+import (
+	"errors"
+
+	objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
+	"github.com/nspcc-dev/neofs-node/pkg/core/object"
+	"go.etcd.io/bbolt"
+)
+
+var errNotFound = errors.New("object not found")
+
+// Get returns object header for specified address.
+func (db *DB) Get(addr *objectSDK.Address) (*object.Object, error) {
+	var obj *object.Object
+
+	if err := db.boltDB.View(func(tx *bbolt.Tx) error {
+		primaryBucket := tx.Bucket(primaryBucket)
+		if primaryBucket == nil {
+			return errNotFound
+		}
+
+		data := primaryBucket.Get(addressKey(addr))
+		if data == nil {
+			return errNotFound
+		}
+
+		var err error
+
+		obj, err = object.FromBytes(data)
+
+		return err
+	}); err != nil {
+		return nil, err
+	}
+
+	return obj, nil
+}

pkg/local_object_storage/metabase/put.go

@@ -0,0 +1,131 @@
+package meta
+
+import (
+	objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
+	v2object "github.com/nspcc-dev/neofs-api-go/v2/object"
+	"github.com/nspcc-dev/neofs-node/pkg/core/object"
+	"github.com/pkg/errors"
+	"go.etcd.io/bbolt"
+)
+
+type bucketItem struct {
+	key, val string
+}
+
+var (
+	primaryBucket = []byte("objects")
+	indexBucket   = []byte("index")
+)
+
+// Put saves object in DB.
+//
+// Object payload expected to be cut.
+func (db *DB) Put(obj *object.Object) error {
+	return db.boltDB.Update(func(tx *bbolt.Tx) error {
+		// create primary bucket (addr: header)
+		primaryBucket, err := tx.CreateBucketIfNotExists(primaryBucket)
+		if err != nil {
+			return errors.Wrapf(err, "(%T) could not create primary bucket", db)
+		}
+
+		data, err := obj.ToV2().StableMarshal(nil)
+		if err != nil {
+			return errors.Wrapf(err, "(%T) could not marshal the object", db)
+		}
+
+		addrKey := addressKey(obj.Address())
+
+		// put header to primary bucket
+		if err := primaryBucket.Put(addrKey, data); err != nil {
+			return errors.Wrapf(err, "(%T) could not put item to primary bucket", db)
+		}
+
+		// create bucket for indices
+		indexBucket, err := tx.CreateBucketIfNotExists(indexBucket)
+		if err != nil {
+			return errors.Wrapf(err, "(%T) could not create index bucket", db)
+		}
+
+		// calculate indexed values for object
+		indices := objectIndices(obj)
+
+		for i := range indices {
+			// create index bucket
+			keyBucket, err := indexBucket.CreateBucketIfNotExists([]byte(indices[i].key))
+			if err != nil {
+				return errors.Wrapf(err, "(%T) could not create bucket for header key", db)
+			}
+
+			// FIXME: here we can get empty slice that could not be the key
+			// Possible solutions:
+			// 1. add prefix byte (0 if empty);
+			v := []byte(indices[i].val)
+
+			// put value to key bucket (it is needed for iteration over all values (Select))
+			if err := keyBucket.Put(keyWithPrefix(v, false), nil); err != nil {
+				return errors.Wrapf(err, "(%T) could not put header value", db)
+			}
+
+			// create address bucket for the value
+			valBucket, err := keyBucket.CreateBucketIfNotExists(keyWithPrefix(v, true))
+			if err != nil {
+				return errors.Wrapf(err, "(%T) could not create bucket for header value", db)
+			}
+
+			// put object address to value bucket
+			if err := valBucket.Put(addrKey, nil); err != nil {
+				return errors.Wrapf(err, "(%T) could not put item to header bucket", db)
+			}
+		}
+
+		return nil
+	})
+}
+
+func keyWithPrefix(key []byte, bucket bool) []byte {
+	b := byte(0)
+	if bucket {
+		b = 1
+	}
+
+	return append([]byte{b}, key...)
+}
+
+func keyWithoutPrefix(key []byte) ([]byte, bool) {
+	return key[1:], key[0] == 1
+}
+
+func addressKey(addr *objectSDK.Address) []byte {
+	return []byte(addr.String())
+}
+
+func objectIndices(obj *object.Object) []bucketItem {
+	as := obj.GetAttributes()
+
+	res := make([]bucketItem, 0, 3+len(as))
+
+	res = append(res,
+		bucketItem{
+			key: v2object.FilterHeaderVersion,
+			val: obj.GetVersion().String(),
+		},
+		bucketItem{
+			key: v2object.FilterHeaderContainerID,
+			val: obj.GetContainerID().String(),
+		},
+		bucketItem{
+			key: v2object.FilterHeaderOwnerID,
+			val: obj.GetOwnerID().String(),
+		},
+		// TODO: add remaining fields after neofs-api#72
+	)
+
+	for _, a := range as {
+		res = append(res, bucketItem{
+			key: a.GetKey(),
+			val: a.GetValue(),
+		})
+	}
+
+	return res
+}

pkg/local_object_storage/metabase/select.go

@@ -0,0 +1,85 @@
+package meta
+
+import (
+	"github.com/nspcc-dev/neofs-api-go/pkg/object"
+	"github.com/pkg/errors"
+	"go.etcd.io/bbolt"
+)
+
+// Select returns list of addresses of objects that match search filters.
+func (db *DB) Select(fs object.SearchFilters) ([]*object.Address, error) {
+	res := make([]*object.Address, 0)
+
+	err := db.boltDB.View(func(tx *bbolt.Tx) error {
+		// get indexed bucket
+		indexBucket := tx.Bucket(indexBucket)
+		if indexBucket == nil {
+			// empty storage
+			return nil
+		}
+
+		// keep addresses that does not match some filter
+		mAddr := make(map[string]struct{})
+
+		for _, f := range fs {
+			matchFunc, ok := db.matchers[f.Operation()]
+			if !ok {
+				return errors.Errorf("no function for matcher %v", f.Operation())
+			}
+
+			key := f.Header()
+
+			// get bucket with values
+			keyBucket := indexBucket.Bucket([]byte(key))
+			if keyBucket == nil {
+				continue
+			}
+
+			fVal := f.Value()
+
+			// iterate over all existing values for the key
+			if err := keyBucket.ForEach(func(k, _ []byte) error {
+				if k, bucket := keyWithoutPrefix(k); !bucket {
+					if !matchFunc(string(k), fVal) {
+						// exclude all addresses with this value
+						return keyBucket.Bucket(keyWithPrefix(k, true)).ForEach(func(k, _ []byte) error {
+							mAddr[string(k)] = struct{}{}
+
+							return nil
+						})
+					}
+				}
+
+				return nil
+			}); err != nil {
+				return errors.Wrapf(err, "(%T) could not iterate bucket %s", db, key)
+			}
+		}
+
+		// get primary bucket
+		primaryBucket := tx.Bucket(primaryBucket)
+		if primaryBucket == nil {
+			// empty storage
+			return nil
+		}
+
+		// iterate over all stored addresses
+		return primaryBucket.ForEach(func(k, v []byte) error {
+			if _, ok := mAddr[string(k)]; ok {
+				return nil
+			}
+
+			addr := object.NewAddress()
+			if err := addr.Parse(string(k)); err != nil {
+				// TODO: storage was broken, so we need to handle it
+				return err
+			}
+
+			res = append(res, addr)
+
+			return nil
+		})
+	})
+
+	return res, err
+}