forked from TrueCloudLab/frostfs-node
[#128] localstorage: Implement primary object metabase
Implement bolt-based metabase that is going to be used in local object storage. Implement Put/Get/Select methods. Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich
Leonard Lyubich
parent
d08c1c76c1
commit
85aacbbb10
7 changed files with 394 additions and 4 deletions
6
go.mod
6
go.mod
|
|
@ -13,7 +13,7 @@ require (
|
|||
github.com/multiformats/go-multiaddr-net v0.1.2 // v0.1.1 => v0.1.2
|
||||
github.com/multiformats/go-multihash v0.0.13 // indirect
|
||||
github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78
|
||||
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201020152448-c8f46f7d9762
|
||||
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201028111149-ac38d13f04ff
|
||||
github.com/nspcc-dev/neofs-crypto v0.3.0
|
||||
github.com/nspcc-dev/tzhash v1.4.0
|
||||
github.com/panjf2000/ants/v2 v2.3.0
|
||||
|
|
@ -25,12 +25,14 @@ require (
|
|||
github.com/spf13/viper v1.7.0
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/valyala/fasthttp v1.9.0
|
||||
go.etcd.io/bbolt v1.3.4
|
||||
go.etcd.io/bbolt v1.3.5
|
||||
go.uber.org/atomic v1.5.1
|
||||
go.uber.org/multierr v1.4.0 // indirect
|
||||
go.uber.org/zap v1.13.0
|
||||
golang.org/x/crypto v0.0.0-20200117160349-530e935923ad // indirect
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
|
||||
golang.org/x/net v0.0.0-20191105084925-a882066a44e0 // indirect
|
||||
golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5 // indirect
|
||||
golang.org/x/tools v0.0.0-20200123022218-593de606220b // indirect
|
||||
google.golang.org/grpc v1.29.1
|
||||
google.golang.org/protobuf v1.23.0
|
||||
|
|
|
|||
11
go.sum
11
go.sum
|
|
@ -71,6 +71,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
|
|||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
|
||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
|
|
@ -270,8 +271,8 @@ github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:
|
|||
github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=
|
||||
github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78 h1:stIa+nBXK8uDY/JZaxIZzAUfkzfaotVw2FbnHxO4aZI=
|
||||
github.com/nspcc-dev/neo-go v0.91.1-pre.0.20200827184617-7560aa345a78/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=
|
||||
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201020152448-c8f46f7d9762 h1:5pCgsBC8XQHX8U+ZBMsX7+87fw7nSMPBwHrewvKLXlk=
|
||||
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201020152448-c8f46f7d9762/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8=
|
||||
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201028111149-ac38d13f04ff h1:TBybhFCjTdSgpw0zGLw7ucjA3gTTLZDfp4D2trcSyT4=
|
||||
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201028111149-ac38d13f04ff/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8=
|
||||
github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA=
|
||||
github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw=
|
||||
github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM=
|
||||
|
|
@ -390,6 +391,8 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
|||
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
|
||||
go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
|
||||
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
|
||||
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
|
|
@ -458,6 +461,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
|
|||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191105084925-a882066a44e0 h1:QPlSTtPE2k6PZPasQUbzuK3p9JbS+vMXYVto8g/yrsg=
|
||||
golang.org/x/net v0.0.0-20191105084925-a882066a44e0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
|
||||
|
|
@ -496,6 +501,8 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=
|
||||
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5 h1:iCaAy5bMeEvwANu3YnJfWwI0kWAGkEa2RXPdweI/ysk=
|
||||
golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
|
|
|
|||
25
pkg/local_object_storage/metabase/db.go
Normal file
25
pkg/local_object_storage/metabase/db.go
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
package meta
|
||||
|
||||
import (
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
// DB represents local metabase of storage node.
|
||||
type DB struct {
|
||||
boltDB *bbolt.DB
|
||||
|
||||
matchers map[object.SearchMatchType]func(string, string) bool
|
||||
}
|
||||
|
||||
// NewDB creates, initializes and returns DB instance.
|
||||
func NewDB(boltDB *bbolt.DB) *DB {
|
||||
return &DB{
|
||||
boltDB: boltDB,
|
||||
matchers: map[object.SearchMatchType]func(string, string) bool{
|
||||
object.MatchStringEqual: func(s string, s2 string) bool {
|
||||
return s == s2
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
102
pkg/local_object_storage/metabase/db_test.go
Normal file
102
pkg/local_object_storage/metabase/db_test.go
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
package meta
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg"
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/container"
|
||||
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/core/object"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/util/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
func testSelect(t *testing.T, db *DB, fs objectSDK.SearchFilters, exp ...*objectSDK.Address) {
|
||||
res, err := db.Select(fs)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, len(exp))
|
||||
|
||||
for i := range exp {
|
||||
require.Contains(t, res, exp[i])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDB(t *testing.T) {
|
||||
version := pkg.NewVersion()
|
||||
version.SetMajor(2)
|
||||
version.SetMinor(1)
|
||||
|
||||
cs := [sha256.Size]byte{}
|
||||
rand.Read(cs[:])
|
||||
|
||||
cid := container.NewID()
|
||||
cid.SetSHA256(cs)
|
||||
|
||||
w, err := owner.NEO3WalletFromPublicKey(&test.DecodeKey(-1).PublicKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
ownerID := owner.NewID()
|
||||
ownerID.SetNeo3Wallet(w)
|
||||
|
||||
rand.Read(cs[:])
|
||||
oid := objectSDK.NewID()
|
||||
oid.SetSHA256(cs)
|
||||
|
||||
obj := object.NewRaw()
|
||||
obj.SetID(oid)
|
||||
obj.SetOwnerID(ownerID)
|
||||
obj.SetContainerID(cid)
|
||||
obj.SetVersion(version)
|
||||
|
||||
k, v := "key", "value"
|
||||
|
||||
a := objectSDK.NewAttribute()
|
||||
a.SetKey(k)
|
||||
a.SetValue(v)
|
||||
|
||||
obj.SetAttributes(a)
|
||||
|
||||
path := "test.db"
|
||||
|
||||
bdb, err := bbolt.Open(path, 0600, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
bdb.Close()
|
||||
os.Remove(path)
|
||||
}()
|
||||
|
||||
db := NewDB(bdb)
|
||||
|
||||
o := obj.Object()
|
||||
|
||||
require.NoError(t, db.Put(o))
|
||||
|
||||
o2, err := db.Get(o.Address())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, o, o2)
|
||||
|
||||
fs := objectSDK.SearchFilters{}
|
||||
|
||||
// filter container ID
|
||||
fs.AddObjectContainerIDFilter(objectSDK.MatchStringEqual, cid)
|
||||
testSelect(t, db, fs, o.Address())
|
||||
|
||||
// filter owner ID
|
||||
fs.AddObjectOwnerIDFilter(objectSDK.MatchStringEqual, ownerID)
|
||||
testSelect(t, db, fs, o.Address())
|
||||
|
||||
// filter attribute
|
||||
fs.AddFilter(k, v, objectSDK.MatchStringEqual)
|
||||
testSelect(t, db, fs, o.Address())
|
||||
|
||||
// filter mismatch
|
||||
fs.AddFilter(k, v+"1", objectSDK.MatchStringEqual)
|
||||
testSelect(t, db, fs)
|
||||
}
|
||||
38
pkg/local_object_storage/metabase/get.go
Normal file
38
pkg/local_object_storage/metabase/get.go
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
package meta
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/core/object"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
var errNotFound = errors.New("object not found")
|
||||
|
||||
// Get returns object header for specified address.
|
||||
func (db *DB) Get(addr *objectSDK.Address) (*object.Object, error) {
|
||||
var obj *object.Object
|
||||
|
||||
if err := db.boltDB.View(func(tx *bbolt.Tx) error {
|
||||
primaryBucket := tx.Bucket(primaryBucket)
|
||||
if primaryBucket == nil {
|
||||
return errNotFound
|
||||
}
|
||||
|
||||
data := primaryBucket.Get(addressKey(addr))
|
||||
if data == nil {
|
||||
return errNotFound
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
obj, err = object.FromBytes(data)
|
||||
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return obj, nil
|
||||
}
|
||||
131
pkg/local_object_storage/metabase/put.go
Normal file
131
pkg/local_object_storage/metabase/put.go
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
package meta
|
||||
|
||||
import (
|
||||
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
v2object "github.com/nspcc-dev/neofs-api-go/v2/object"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/core/object"
|
||||
"github.com/pkg/errors"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
type bucketItem struct {
|
||||
key, val string
|
||||
}
|
||||
|
||||
var (
|
||||
primaryBucket = []byte("objects")
|
||||
indexBucket = []byte("index")
|
||||
)
|
||||
|
||||
// Put saves object in DB.
|
||||
//
|
||||
// Object payload expected to be cut.
|
||||
func (db *DB) Put(obj *object.Object) error {
|
||||
return db.boltDB.Update(func(tx *bbolt.Tx) error {
|
||||
// create primary bucket (addr: header)
|
||||
primaryBucket, err := tx.CreateBucketIfNotExists(primaryBucket)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "(%T) could not create primary bucket", db)
|
||||
}
|
||||
|
||||
data, err := obj.ToV2().StableMarshal(nil)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "(%T) could not marshal the object", db)
|
||||
}
|
||||
|
||||
addrKey := addressKey(obj.Address())
|
||||
|
||||
// put header to primary bucket
|
||||
if err := primaryBucket.Put(addrKey, data); err != nil {
|
||||
return errors.Wrapf(err, "(%T) could not put item to primary bucket", db)
|
||||
}
|
||||
|
||||
// create bucket for indices
|
||||
indexBucket, err := tx.CreateBucketIfNotExists(indexBucket)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "(%T) could not create index bucket", db)
|
||||
}
|
||||
|
||||
// calculate indexed values for object
|
||||
indices := objectIndices(obj)
|
||||
|
||||
for i := range indices {
|
||||
// create index bucket
|
||||
keyBucket, err := indexBucket.CreateBucketIfNotExists([]byte(indices[i].key))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "(%T) could not create bucket for header key", db)
|
||||
}
|
||||
|
||||
// FIXME: here we can get empty slice that could not be the key
|
||||
// Possible solutions:
|
||||
// 1. add prefix byte (0 if empty);
|
||||
v := []byte(indices[i].val)
|
||||
|
||||
// put value to key bucket (it is needed for iteration over all values (Select))
|
||||
if err := keyBucket.Put(keyWithPrefix(v, false), nil); err != nil {
|
||||
return errors.Wrapf(err, "(%T) could not put header value", db)
|
||||
}
|
||||
|
||||
// create address bucket for the value
|
||||
valBucket, err := keyBucket.CreateBucketIfNotExists(keyWithPrefix(v, true))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "(%T) could not create bucket for header value", db)
|
||||
}
|
||||
|
||||
// put object address to value bucket
|
||||
if err := valBucket.Put(addrKey, nil); err != nil {
|
||||
return errors.Wrapf(err, "(%T) could not put item to header bucket", db)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func keyWithPrefix(key []byte, bucket bool) []byte {
|
||||
b := byte(0)
|
||||
if bucket {
|
||||
b = 1
|
||||
}
|
||||
|
||||
return append([]byte{b}, key...)
|
||||
}
|
||||
|
||||
func keyWithoutPrefix(key []byte) ([]byte, bool) {
|
||||
return key[1:], key[0] == 1
|
||||
}
|
||||
|
||||
func addressKey(addr *objectSDK.Address) []byte {
|
||||
return []byte(addr.String())
|
||||
}
|
||||
|
||||
func objectIndices(obj *object.Object) []bucketItem {
|
||||
as := obj.GetAttributes()
|
||||
|
||||
res := make([]bucketItem, 0, 3+len(as))
|
||||
|
||||
res = append(res,
|
||||
bucketItem{
|
||||
key: v2object.FilterHeaderVersion,
|
||||
val: obj.GetVersion().String(),
|
||||
},
|
||||
bucketItem{
|
||||
key: v2object.FilterHeaderContainerID,
|
||||
val: obj.GetContainerID().String(),
|
||||
},
|
||||
bucketItem{
|
||||
key: v2object.FilterHeaderOwnerID,
|
||||
val: obj.GetOwnerID().String(),
|
||||
},
|
||||
// TODO: add remaining fields after neofs-api#72
|
||||
)
|
||||
|
||||
for _, a := range as {
|
||||
res = append(res, bucketItem{
|
||||
key: a.GetKey(),
|
||||
val: a.GetValue(),
|
||||
})
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
85
pkg/local_object_storage/metabase/select.go
Normal file
85
pkg/local_object_storage/metabase/select.go
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
package meta
|
||||
|
||||
import (
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
"github.com/pkg/errors"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
// Select returns list of addresses of objects that match search filters.
|
||||
func (db *DB) Select(fs object.SearchFilters) ([]*object.Address, error) {
|
||||
res := make([]*object.Address, 0)
|
||||
|
||||
err := db.boltDB.View(func(tx *bbolt.Tx) error {
|
||||
// get indexed bucket
|
||||
indexBucket := tx.Bucket(indexBucket)
|
||||
if indexBucket == nil {
|
||||
// empty storage
|
||||
return nil
|
||||
}
|
||||
|
||||
// keep addresses that does not match some filter
|
||||
mAddr := make(map[string]struct{})
|
||||
|
||||
for _, f := range fs {
|
||||
matchFunc, ok := db.matchers[f.Operation()]
|
||||
if !ok {
|
||||
return errors.Errorf("no function for matcher %v", f.Operation())
|
||||
}
|
||||
|
||||
key := f.Header()
|
||||
|
||||
// get bucket with values
|
||||
keyBucket := indexBucket.Bucket([]byte(key))
|
||||
if keyBucket == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
fVal := f.Value()
|
||||
|
||||
// iterate over all existing values for the key
|
||||
if err := keyBucket.ForEach(func(k, _ []byte) error {
|
||||
if k, bucket := keyWithoutPrefix(k); !bucket {
|
||||
if !matchFunc(string(k), fVal) {
|
||||
// exclude all addresses with this value
|
||||
return keyBucket.Bucket(keyWithPrefix(k, true)).ForEach(func(k, _ []byte) error {
|
||||
mAddr[string(k)] = struct{}{}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return errors.Wrapf(err, "(%T) could not iterate bucket %s", db, key)
|
||||
}
|
||||
}
|
||||
|
||||
// get primary bucket
|
||||
primaryBucket := tx.Bucket(primaryBucket)
|
||||
if primaryBucket == nil {
|
||||
// empty storage
|
||||
return nil
|
||||
}
|
||||
|
||||
// iterate over all stored addresses
|
||||
return primaryBucket.ForEach(func(k, v []byte) error {
|
||||
if _, ok := mAddr[string(k)]; ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
addr := object.NewAddress()
|
||||
if err := addr.Parse(string(k)); err != nil {
|
||||
// TODO: storage was broken, so we need to handle it
|
||||
return err
|
||||
}
|
||||
|
||||
res = append(res, addr)
|
||||
|
||||
return nil
|
||||
})
|
||||
})
|
||||
|
||||
return res, err
|
||||
}
|
||||
Loading…
Reference in a new issue