[#361] metabase: Support new match types

Support STRING_NOT_EQUAL and NOT_PRESENT match types.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-02-02 00:00:40 +03:00 committed by Alex Vanin
parent 487c9b7589
commit ffbf6b922f
5 changed files with 432 additions and 65 deletions

2
go.mod
View file

@ -16,7 +16,7 @@ require (
github.com/multiformats/go-multihash v0.0.13 // indirect github.com/multiformats/go-multihash v0.0.13 // indirect
github.com/nspcc-dev/hrw v1.0.9 github.com/nspcc-dev/hrw v1.0.9
github.com/nspcc-dev/neo-go v0.92.0 github.com/nspcc-dev/neo-go v0.92.0
github.com/nspcc-dev/neofs-api-go v1.22.3-0.20210127171042-f654094edb2e github.com/nspcc-dev/neofs-api-go v1.22.3-0.20210202125050-01c7e17b081d
github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/nspcc-dev/neofs-crypto v0.3.0
github.com/nspcc-dev/tzhash v1.4.0 github.com/nspcc-dev/tzhash v1.4.0
github.com/panjf2000/ants/v2 v2.3.0 github.com/panjf2000/ants/v2 v2.3.0

4
go.sum
View file

@ -280,8 +280,8 @@ github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:
github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=
github.com/nspcc-dev/neo-go v0.92.0 h1:iKHpKLzpwE6RSXnQb0BoYWi+H1P/hNyQbMpPG0mY57Q= github.com/nspcc-dev/neo-go v0.92.0 h1:iKHpKLzpwE6RSXnQb0BoYWi+H1P/hNyQbMpPG0mY57Q=
github.com/nspcc-dev/neo-go v0.92.0/go.mod h1:L7PyTzjK1j/PCAxvbKiVFkCMZDvsv82JbXlPxaH1t0Q= github.com/nspcc-dev/neo-go v0.92.0/go.mod h1:L7PyTzjK1j/PCAxvbKiVFkCMZDvsv82JbXlPxaH1t0Q=
github.com/nspcc-dev/neofs-api-go v1.22.3-0.20210127171042-f654094edb2e h1:naIdoA6cWsD8ltPoWOqqRO9Su86PQHTma18jlq7tHEo= github.com/nspcc-dev/neofs-api-go v1.22.3-0.20210202125050-01c7e17b081d h1:5zqGSPoNBIAhByjZEEDrsnSAfLUgCNBNVyWVGsbrj28=
github.com/nspcc-dev/neofs-api-go v1.22.3-0.20210127171042-f654094edb2e/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-api-go v1.22.3-0.20210202125050-01c7e17b081d/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8=
github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA=
github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw=
github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM=

View file

@ -54,23 +54,32 @@ func New(opts ...Option) *DB {
return &DB{ return &DB{
cfg: c, cfg: c,
matchers: map[object.SearchMatchType]func(string, []byte, string) bool{ matchers: map[object.SearchMatchType]func(string, []byte, string) bool{
object.MatchUnknown: unknownMatcher, object.MatchUnknown: unknownMatcher,
object.MatchStringEqual: stringEqualMatcher, object.MatchStringEqual: stringEqualMatcher,
object.MatchStringNotEqual: stringNotEqualMatcher,
}, },
} }
} }
func stringEqualMatcher(key string, objVal []byte, filterVal string) bool { func stringifyValue(key string, objVal []byte) string {
switch key { switch key {
default: default:
return string(objVal) == filterVal return string(objVal)
case v2object.FilterHeaderPayloadHash, v2object.FilterHeaderHomomorphicHash: case v2object.FilterHeaderPayloadHash, v2object.FilterHeaderHomomorphicHash:
return hex.EncodeToString(objVal) == filterVal return hex.EncodeToString(objVal)
case v2object.FilterHeaderCreationEpoch, v2object.FilterHeaderPayloadLength: case v2object.FilterHeaderCreationEpoch, v2object.FilterHeaderPayloadLength:
return strconv.FormatUint(binary.LittleEndian.Uint64(objVal), 10) == filterVal return strconv.FormatUint(binary.LittleEndian.Uint64(objVal), 10)
} }
} }
func stringEqualMatcher(key string, objVal []byte, filterVal string) bool {
return stringifyValue(key, objVal) == filterVal
}
func stringNotEqualMatcher(key string, objVal []byte, filterVal string) bool {
return stringifyValue(key, objVal) != filterVal
}
func unknownMatcher(_ string, _ []byte, _ string) bool { func unknownMatcher(_ string, _ []byte, _ string) bool {
return false return false
} }

View file

@ -3,6 +3,7 @@ package meta
import ( import (
"encoding/binary" "encoding/binary"
"fmt" "fmt"
"strings"
"github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/container"
"github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/object"
@ -88,6 +89,11 @@ func (db *DB) selectObjects(tx *bbolt.Tx, cid *container.ID, fs object.SearchFil
return nil, ErrMissingContainerID return nil, ErrMissingContainerID
} }
// TODO: consider the option of moving this check to a level higher than the metabase
if blindlyProcess(fs) {
return nil, nil
}
group, err := groupFilters(fs) group, err := groupFilters(fs)
if err != nil { if err != nil {
return nil, err return nil, err
@ -181,7 +187,7 @@ func (db *DB) selectFastFilter(
switch f.Header() { switch f.Header() {
case v2object.FilterHeaderObjectID: case v2object.FilterHeaderObjectID:
db.selectObjectID(tx, f, prefix, to, fNum) db.selectObjectID(tx, f, cid, to, fNum)
case v2object.FilterHeaderOwnerID: case v2object.FilterHeaderOwnerID:
bucketName := ownerBucketName(cid) bucketName := ownerBucketName(cid)
db.selectFromFKBT(tx, bucketName, f, prefix, to, fNum) db.selectFromFKBT(tx, bucketName, f, prefix, to, fNum)
@ -189,26 +195,9 @@ func (db *DB) selectFastFilter(
bucketName := payloadHashBucketName(cid) bucketName := payloadHashBucketName(cid)
db.selectFromList(tx, bucketName, f, prefix, to, fNum) db.selectFromList(tx, bucketName, f, prefix, to, fNum)
case v2object.FilterHeaderObjectType: case v2object.FilterHeaderObjectType:
var bucketName []byte for _, bucketName := range bucketNamesForType(cid, f.Operation(), f.Value()) {
switch f.Value() { // do it better after https://github.com/nspcc-dev/neofs-api/issues/84
case "Regular":
bucketName = primaryBucketName(cid)
selectAllFromBucket(tx, bucketName, prefix, to, fNum) selectAllFromBucket(tx, bucketName, prefix, to, fNum)
bucketName = parentBucketName(cid)
case "Tombstone":
bucketName = tombstoneBucketName(cid)
case "StorageGroup":
bucketName = storageGroupBucketName(cid)
default:
db.log.Debug("unknown object type", zap.String("type", f.Value()))
return
} }
selectAllFromBucket(tx, bucketName, prefix, to, fNum)
case v2object.FilterHeaderParent: case v2object.FilterHeaderParent:
bucketName := parentBucketName(cid) bucketName := parentBucketName(cid)
db.selectFromList(tx, bucketName, f, prefix, to, fNum) db.selectFromList(tx, bucketName, f, prefix, to, fNum)
@ -223,10 +212,57 @@ func (db *DB) selectFastFilter(
selectAllFromBucket(tx, storageGroupBucketName(cid), prefix, to, fNum) selectAllFromBucket(tx, storageGroupBucketName(cid), prefix, to, fNum)
default: // user attribute default: // user attribute
bucketName := attributeBucketName(cid, f.Header()) bucketName := attributeBucketName(cid, f.Header())
db.selectFromFKBT(tx, bucketName, f, prefix, to, fNum)
if f.Operation() == object.MatchNotPresent {
selectOutsideFKBT(tx, allBucketNames(cid), bucketName, f, prefix, to, fNum)
} else {
db.selectFromFKBT(tx, bucketName, f, prefix, to, fNum)
}
} }
} }
// TODO: move to DB struct
var mBucketNaming = map[string][]func(*container.ID) []byte{
v2object.TypeRegular.String(): {primaryBucketName, parentBucketName},
v2object.TypeTombstone.String(): {tombstoneBucketName},
v2object.TypeStorageGroup.String(): {storageGroupBucketName},
}
func allBucketNames(cid *container.ID) (names [][]byte) {
for _, fns := range mBucketNaming {
for _, fn := range fns {
names = append(names, fn(cid))
}
}
return
}
func bucketNamesForType(cid *container.ID, mType object.SearchMatchType, typeVal string) (names [][]byte) {
appendNames := func(key string) {
fns, ok := mBucketNaming[key]
if ok {
for _, fn := range fns {
names = append(names, fn(cid))
}
}
}
switch mType {
default:
case object.MatchStringNotEqual:
for key := range mBucketNaming {
if key != typeVal {
appendNames(key)
}
}
case object.MatchStringEqual:
appendNames(typeVal)
}
return
}
// selectFromList looks into <fkbt> index to find list of addresses to add in // selectFromList looks into <fkbt> index to find list of addresses to add in
// resulting cache. // resulting cache.
func (db *DB) selectFromFKBT( func (db *DB) selectFromFKBT(
@ -271,6 +307,54 @@ func (db *DB) selectFromFKBT(
} }
} }
// selectOutsideFKBT looks into all incl buckets to find list of addresses outside <fkbt> to add in
// resulting cache.
func selectOutsideFKBT(
tx *bbolt.Tx,
incl [][]byte, // buckets
name []byte, // fkbt root bucket name
f object.SearchFilter, // filter for operation and value
prefix string, // prefix to create addr from oid in index
to map[string]int, // resulting cache
fNum int, // index of filter
) {
mExcl := make(map[string]struct{})
bktExcl := tx.Bucket(name)
if bktExcl != nil {
_ = bktExcl.ForEach(func(k, _ []byte) error {
exclBktLeaf := bktExcl.Bucket(k)
if exclBktLeaf == nil {
return nil
}
return exclBktLeaf.ForEach(func(k, _ []byte) error {
addr := prefix + string(k)
mExcl[addr] = struct{}{}
return nil
})
})
}
for i := range incl {
bktIncl := tx.Bucket(incl[i])
if bktIncl == nil {
continue
}
_ = bktIncl.ForEach(func(k, _ []byte) error {
addr := prefix + string(k)
if _, ok := mExcl[addr]; !ok {
markAddressInCache(to, fNum, addr)
}
return nil
})
}
}
// selectFromList looks into <list> index to find list of addresses to add in // selectFromList looks into <list> index to find list of addresses to add in
// resulting cache. // resulting cache.
func (db *DB) selectFromList( func (db *DB) selectFromList(
@ -286,21 +370,50 @@ func (db *DB) selectFromList(
return return
} }
switch f.Operation() { var (
lst [][]byte
err error
)
switch op := f.Operation(); op {
case object.MatchStringEqual: case object.MatchStringEqual:
lst, err = decodeList(bkt.Get(bucketKeyHelper(f.Header(), f.Value())))
if err != nil {
db.log.Debug("can't decode list bucket leaf", zap.String("error", err.Error()))
return
}
default: default:
db.log.Debug("unknown operation", zap.Uint32("operation", uint32(f.Operation()))) fMatch, ok := db.matchers[op]
if !ok {
db.log.Debug("unknown operation", zap.Uint32("operation", uint32(op)))
return return
} }
// warning: it works only for MatchStringEQ, for NotEQ you should iterate over if err = bkt.ForEach(func(key, val []byte) error {
// bkt and apply matchFunc, don't forget to implement this when it will be if !fMatch(f.Header(), key, f.Value()) {
// needed. Right now it is not efficient to iterate over bucket return nil
// when there is only MatchStringEQ. }
lst, err := decodeList(bkt.Get(bucketKeyHelper(f.Header(), f.Value())))
if err != nil { l, err := decodeList(val)
db.log.Debug("can't decode list bucket leaf", zap.String("error", err.Error())) if err != nil {
db.log.Debug("can't decode list bucket leaf",
zap.String("error", err.Error()),
)
return err
}
lst = append(lst, l...)
return nil
}); err != nil {
db.log.Debug("can't iterate over the bucket",
zap.String("error", err.Error()),
)
return
}
} }
for i := range lst { for i := range lst {
@ -313,36 +426,65 @@ func (db *DB) selectFromList(
func (db *DB) selectObjectID( func (db *DB) selectObjectID(
tx *bbolt.Tx, tx *bbolt.Tx,
f object.SearchFilter, f object.SearchFilter,
prefix string, cid *container.ID,
to map[string]int, // resulting cache to map[string]int, // resulting cache
fNum int, // index of filter fNum int, // index of filter
) { ) {
switch f.Operation() { prefix := cid.String() + "/"
appendOID := func(oid string) {
addrStr := prefix + string(oid)
addr := object.NewAddress()
err := addr.Parse(addrStr)
if err != nil {
db.log.Debug("can't decode object id address",
zap.String("addr", addrStr),
zap.String("error", err.Error()))
return
}
ok, err := db.exists(tx, addr)
if (err == nil && ok) || errors.As(err, &splitInfoError) {
markAddressInCache(to, fNum, addrStr)
}
}
switch op := f.Operation(); op {
case object.MatchStringEqual: case object.MatchStringEqual:
appendOID(f.Value())
default: default:
db.log.Debug("unknown operation", zap.Uint32("operation", uint32(f.Operation()))) fMatch, ok := db.matchers[op]
if !ok {
db.log.Debug("unknown operation",
zap.Uint32("operation", uint32(f.Operation())),
)
return return
} }
// warning: it is in-place optimization and works only for MatchStringEQ, for _, bucketName := range bucketNamesForType(cid, object.MatchStringNotEqual, "") {
// for NotEQ you should iterate over bkt and apply matchFunc // copy-paste from DB.selectAllFrom
bkt := tx.Bucket(bucketName)
if bkt == nil {
return
}
addrStr := prefix + f.Value() err := bkt.ForEach(func(k, v []byte) error {
addr := object.NewAddress() if oid := string(k); fMatch(f.Header(), k, f.Value()) {
appendOID(oid)
}
err := addr.Parse(addrStr) return nil
if err != nil { })
db.log.Debug("can't decode object id address", if err != nil {
zap.String("addr", addrStr), db.log.Debug("could not iterate over the buckets",
zap.String("error", err.Error())) zap.String("error", err.Error()),
)
return }
} }
ok, err := db.exists(tx, addr)
if (err == nil && ok) || errors.As(err, &splitInfoError) {
markAddressInCache(to, fNum, addrStr)
} }
} }
@ -425,3 +567,23 @@ func markAddressInCache(cache map[string]int, fNum int, addr string) {
cache[addr] = num + 1 cache[addr] = num + 1
} }
} }
// returns true if query leads to a deliberately empty result.
func blindlyProcess(fs object.SearchFilters) bool {
for i := range fs {
if fs[i].Operation() == object.MatchNotPresent && isSystemKey(fs[i].Header()) {
return true
}
// TODO: check other cases
// e.g. (a == b) && (a != b)
}
return false
}
// returns true if string key is a reserved system filter key.
func isSystemKey(key string) bool {
// FIXME: version-dependent approach
return strings.HasPrefix(key, v2object.ReservedFilterPrefix)
}

View file

@ -48,6 +48,10 @@ func TestDB_SelectUserAttributes(t *testing.T) {
fs.AddFilter("x", "y", objectSDK.MatchStringEqual) fs.AddFilter("x", "y", objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, raw1.Object().Address()) testSelect(t, db, cid, fs, raw1.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter("x", "y", objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs, raw2.Object().Address())
fs = objectSDK.SearchFilters{} fs = objectSDK.SearchFilters{}
fs.AddFilter("a", "b", objectSDK.MatchStringEqual) fs.AddFilter("a", "b", objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, raw3.Object().Address()) testSelect(t, db, cid, fs, raw3.Object().Address())
@ -57,6 +61,22 @@ func TestDB_SelectUserAttributes(t *testing.T) {
testSelect(t, db, cid, fs) testSelect(t, db, cid, fs)
fs = objectSDK.SearchFilters{} fs = objectSDK.SearchFilters{}
fs.AddFilter("foo", "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs, raw3.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter("a", "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs, raw1.Object().Address(), raw2.Object().Address())
fs = objectSDK.SearchFilters{}
testSelect(t, db, cid, fs,
raw1.Object().Address(),
raw2.Object().Address(),
raw3.Object().Address(),
)
fs = objectSDK.SearchFilters{}
fs.AddFilter("key", "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs, testSelect(t, db, cid, fs,
raw1.Object().Address(), raw1.Object().Address(),
raw2.Object().Address(), raw2.Object().Address(),
@ -114,6 +134,10 @@ func TestDB_SelectRootPhyParent(t *testing.T) {
small.Object().Address(), small.Object().Address(),
parent.Object().Address(), parent.Object().Address(),
) )
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterPropertyRoot, "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
}) })
t.Run("phy objects", func(t *testing.T) { t.Run("phy objects", func(t *testing.T) {
@ -127,11 +151,15 @@ func TestDB_SelectRootPhyParent(t *testing.T) {
rightChild.Object().Address(), rightChild.Object().Address(),
link.Object().Address(), link.Object().Address(),
) )
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterPropertyPhy, "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
}) })
t.Run("regular objects", func(t *testing.T) { t.Run("regular objects", func(t *testing.T) {
fs := objectSDK.SearchFilters{} fs := objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderObjectType, "Regular", objectSDK.MatchStringEqual) fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeRegular.String(), objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, testSelect(t, db, cid, fs,
small.Object().Address(), small.Object().Address(),
leftChild.Object().Address(), leftChild.Object().Address(),
@ -139,18 +167,59 @@ func TestDB_SelectRootPhyParent(t *testing.T) {
link.Object().Address(), link.Object().Address(),
parent.Object().Address(), parent.Object().Address(),
) )
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeRegular.String(), objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs,
ts.Object().Address(),
sg.Object().Address(),
)
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderObjectType, "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
}) })
t.Run("tombstone objects", func(t *testing.T) { t.Run("tombstone objects", func(t *testing.T) {
fs := objectSDK.SearchFilters{} fs := objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderObjectType, "Tombstone", objectSDK.MatchStringEqual) fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeTombstone.String(), objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, ts.Object().Address()) testSelect(t, db, cid, fs, ts.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeTombstone.String(), objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs,
small.Object().Address(),
leftChild.Object().Address(),
rightChild.Object().Address(),
link.Object().Address(),
parent.Object().Address(),
sg.Object().Address(),
)
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderObjectType, "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
}) })
t.Run("storage group objects", func(t *testing.T) { t.Run("storage group objects", func(t *testing.T) {
fs := objectSDK.SearchFilters{} fs := objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderObjectType, "StorageGroup", objectSDK.MatchStringEqual) fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeStorageGroup.String(), objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, sg.Object().Address()) testSelect(t, db, cid, fs, sg.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeStorageGroup.String(), objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs,
small.Object().Address(),
leftChild.Object().Address(),
rightChild.Object().Address(),
link.Object().Address(),
parent.Object().Address(),
ts.Object().Address(),
)
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderObjectType, "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
}) })
t.Run("objects with parent", func(t *testing.T) { t.Run("objects with parent", func(t *testing.T) {
@ -163,6 +232,10 @@ func TestDB_SelectRootPhyParent(t *testing.T) {
rightChild.Object().Address(), rightChild.Object().Address(),
link.Object().Address(), link.Object().Address(),
) )
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderParent, "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
}) })
t.Run("all objects", func(t *testing.T) { t.Run("all objects", func(t *testing.T) {
@ -232,6 +305,20 @@ func TestDB_SelectPayloadHash(t *testing.T) {
objectSDK.MatchStringEqual) objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, raw1.Object().Address()) testSelect(t, db, cid, fs, raw1.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderPayloadHash,
hex.EncodeToString(raw1.PayloadChecksum().Sum()),
objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs, raw2.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderPayloadHash,
"",
objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
} }
func TestDB_SelectWithSlowFilters(t *testing.T) { func TestDB_SelectWithSlowFilters(t *testing.T) {
@ -268,6 +355,20 @@ func TestDB_SelectWithSlowFilters(t *testing.T) {
objectSDK.MatchStringEqual) objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, raw1.Object().Address()) testSelect(t, db, cid, fs, raw1.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderHomomorphicHash,
hex.EncodeToString(raw1.PayloadHomomorphicHash().Sum()),
objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs, raw2.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderHomomorphicHash,
"",
objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
}) })
t.Run("object with payload length", func(t *testing.T) { t.Run("object with payload length", func(t *testing.T) {
@ -275,6 +376,16 @@ func TestDB_SelectWithSlowFilters(t *testing.T) {
fs.AddFilter(v2object.FilterHeaderPayloadLength, "20", objectSDK.MatchStringEqual) fs.AddFilter(v2object.FilterHeaderPayloadLength, "20", objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, raw2.Object().Address()) testSelect(t, db, cid, fs, raw2.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderPayloadLength, "20", objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs, raw1.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderPayloadLength, "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
}) })
t.Run("object with creation epoch", func(t *testing.T) { t.Run("object with creation epoch", func(t *testing.T) {
@ -282,12 +393,30 @@ func TestDB_SelectWithSlowFilters(t *testing.T) {
fs.AddFilter(v2object.FilterHeaderCreationEpoch, "11", objectSDK.MatchStringEqual) fs.AddFilter(v2object.FilterHeaderCreationEpoch, "11", objectSDK.MatchStringEqual)
testSelect(t, db, cid, fs, raw1.Object().Address()) testSelect(t, db, cid, fs, raw1.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderCreationEpoch, "11", objectSDK.MatchStringNotEqual)
testSelect(t, db, cid, fs, raw2.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderCreationEpoch, "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
}) })
t.Run("object with version", func(t *testing.T) { t.Run("object with version", func(t *testing.T) {
fs := objectSDK.SearchFilters{} fs := objectSDK.SearchFilters{}
fs.AddObjectVersionFilter(objectSDK.MatchStringEqual, v21) fs.AddObjectVersionFilter(objectSDK.MatchStringEqual, v21)
testSelect(t, db, cid, fs, raw2.Object().Address()) testSelect(t, db, cid, fs, raw2.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddObjectVersionFilter(objectSDK.MatchStringNotEqual, v21)
testSelect(t, db, cid, fs, raw1.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddObjectVersionFilter(objectSDK.MatchNotPresent, nil)
testSelect(t, db, cid, fs)
}) })
} }
@ -318,6 +447,12 @@ func TestDB_SelectObjectID(t *testing.T) {
err = putBig(db, sg.Object()) err = putBig(db, sg.Object())
require.NoError(t, err) require.NoError(t, err)
t.Run("not present", func(t *testing.T) {
fs := objectSDK.SearchFilters{}
fs.AddObjectIDFilter(objectSDK.MatchNotPresent, nil)
testSelect(t, db, cid, fs)
})
t.Run("not found objects", func(t *testing.T) { t.Run("not found objects", func(t *testing.T) {
raw := generateRawObjectWithCID(t, cid) raw := generateRawObjectWithCID(t, cid)
@ -325,30 +460,72 @@ func TestDB_SelectObjectID(t *testing.T) {
fs.AddObjectIDFilter(objectSDK.MatchStringEqual, raw.ID()) fs.AddObjectIDFilter(objectSDK.MatchStringEqual, raw.ID())
testSelect(t, db, cid, fs) testSelect(t, db, cid, fs)
fs = objectSDK.SearchFilters{}
fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, raw.ID())
testSelect(t, db, cid, fs,
regular.Object().Address(),
parent.Object().Address(),
sg.Object().Address(),
ts.Object().Address(),
)
}) })
t.Run("regular objects", func(t *testing.T) { t.Run("regular objects", func(t *testing.T) {
fs := objectSDK.SearchFilters{} fs := objectSDK.SearchFilters{}
fs.AddObjectIDFilter(objectSDK.MatchStringEqual, regular.ID()) fs.AddObjectIDFilter(objectSDK.MatchStringEqual, regular.ID())
testSelect(t, db, cid, fs, regular.Object().Address()) testSelect(t, db, cid, fs, regular.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, regular.ID())
testSelect(t, db, cid, fs,
parent.Object().Address(),
sg.Object().Address(),
ts.Object().Address(),
)
}) })
t.Run("tombstone objects", func(t *testing.T) { t.Run("tombstone objects", func(t *testing.T) {
fs := objectSDK.SearchFilters{} fs := objectSDK.SearchFilters{}
fs.AddObjectIDFilter(objectSDK.MatchStringEqual, ts.ID()) fs.AddObjectIDFilter(objectSDK.MatchStringEqual, ts.ID())
testSelect(t, db, cid, fs, ts.Object().Address()) testSelect(t, db, cid, fs, ts.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, ts.ID())
testSelect(t, db, cid, fs,
regular.Object().Address(),
parent.Object().Address(),
sg.Object().Address(),
)
}) })
t.Run("storage group objects", func(t *testing.T) { t.Run("storage group objects", func(t *testing.T) {
fs := objectSDK.SearchFilters{} fs := objectSDK.SearchFilters{}
fs.AddObjectIDFilter(objectSDK.MatchStringEqual, sg.ID()) fs.AddObjectIDFilter(objectSDK.MatchStringEqual, sg.ID())
testSelect(t, db, cid, fs, sg.Object().Address()) testSelect(t, db, cid, fs, sg.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, sg.ID())
testSelect(t, db, cid, fs,
regular.Object().Address(),
parent.Object().Address(),
ts.Object().Address(),
)
}) })
t.Run("storage group objects", func(t *testing.T) { t.Run("parent objects", func(t *testing.T) {
fs := objectSDK.SearchFilters{} fs := objectSDK.SearchFilters{}
fs.AddObjectIDFilter(objectSDK.MatchStringEqual, parent.ID()) fs.AddObjectIDFilter(objectSDK.MatchStringEqual, parent.ID())
testSelect(t, db, cid, fs, parent.Object().Address()) testSelect(t, db, cid, fs, parent.Object().Address())
fs = objectSDK.SearchFilters{}
fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, parent.ID())
testSelect(t, db, cid, fs,
regular.Object().Address(),
sg.Object().Address(),
ts.Object().Address(),
)
}) })
} }
@ -373,6 +550,12 @@ func TestDB_SelectSplitID(t *testing.T) {
require.NoError(t, putBig(db, child2.Object())) require.NoError(t, putBig(db, child2.Object()))
require.NoError(t, putBig(db, child3.Object())) require.NoError(t, putBig(db, child3.Object()))
t.Run("not present", func(t *testing.T) {
fs := objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderSplitID, "", objectSDK.MatchNotPresent)
testSelect(t, db, cid, fs)
})
t.Run("split id", func(t *testing.T) { t.Run("split id", func(t *testing.T) {
fs := objectSDK.SearchFilters{} fs := objectSDK.SearchFilters{}
fs.AddFilter(v2object.FilterHeaderSplitID, split1.String(), objectSDK.MatchStringEqual) fs.AddFilter(v2object.FilterHeaderSplitID, split1.String(), objectSDK.MatchStringEqual)
@ -423,6 +606,19 @@ func TestDB_SelectContainerID(t *testing.T) {
obj1.Object().Address(), obj1.Object().Address(),
obj2.Object().Address(), obj2.Object().Address(),
) )
fs = objectSDK.SearchFilters{}
fs.AddObjectContainerIDFilter(objectSDK.MatchStringNotEqual, cid)
testSelect(t, db, cid, fs,
obj1.Object().Address(),
obj2.Object().Address(),
)
fs = objectSDK.SearchFilters{}
fs.AddObjectContainerIDFilter(objectSDK.MatchNotPresent, cid)
testSelect(t, db, cid, fs)
}) })
t.Run("not same cid", func(t *testing.T) { t.Run("not same cid", func(t *testing.T) {