[#1634] meta: Add epoch state

It allows performing expiration checks on the stored objects.

Signed-off-by: Pavel Karpy <carpawell@nspcc.ru>
This commit is contained in:
Pavel Karpy 2022-07-27 21:34:25 +03:00 committed by Pavel Karpy
parent a97dee008c
commit 9aba0ba512
9 changed files with 76 additions and 7 deletions

View file

@ -473,6 +473,7 @@ func initShardOptions(c *cfg) {
meta.WithBoltDBOptions(&bbolt.Options{
Timeout: 100 * time.Millisecond,
}),
meta.WithEpochState(c.cfgNetmap.state),
),
shard.WithPiloramaOptions(piloramaOpts...),
shard.WithWriteCache(writeCacheCfg.Enabled()),

View file

@ -2,6 +2,7 @@ package engine
import (
"fmt"
"math"
"os"
"path/filepath"
"testing"
@ -25,6 +26,12 @@ import (
"go.uber.org/zap"
)
type epochState struct{}
func (s epochState) CurrentEpoch() uint64 {
return math.MaxUint64
}
func BenchmarkExists(b *testing.B) {
b.Run("2 shards", func(b *testing.B) {
benchmarkExists(b, 2)
@ -104,6 +111,7 @@ func testNewShard(t testing.TB, id int) *shard.Shard {
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.metabase", id))),
meta.WithPermissions(0700),
meta.WithEpochState(epochState{}),
))
require.NoError(t, s.Open())
@ -125,6 +133,7 @@ func testEngineFromShardOpts(t *testing.T, num int, extraOpts func(int) []shard.
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("metabase%d", i))),
meta.WithPermissions(0700),
meta.WithEpochState(epochState{}),
),
shard.WithPiloramaOptions(
pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("pilorama%d", i)))),

View file

@ -51,7 +51,9 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32)
blobstor.WithRootPerm(0700)),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", i))),
meta.WithPermissions(0700)),
meta.WithPermissions(0700),
meta.WithEpochState(epochState{}),
),
shard.WithPiloramaOptions(
pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", i))),
pilorama.WithPerm(0700)))

View file

@ -24,6 +24,13 @@ type matcher struct {
matchBucket func(*bbolt.Bucket, string, string, func([]byte, []byte) error) error
}
// EpochState is an interface that provides access to the
// current epoch number.
type EpochState interface {
// CurrentEpoch must return current epoch height.
CurrentEpoch() uint64
}
// DB represents local metabase of storage node.
type DB struct {
*cfg
@ -50,6 +57,8 @@ type cfg struct {
info Info
log *logger.Logger
epochState EpochState
}
func defaultCfg() *cfg {
@ -71,6 +80,10 @@ func New(opts ...Option) *DB {
opts[i](c)
}
if c.epochState == nil {
panic("metabase: epoch state is not specified")
}
return &DB{
cfg: c,
matchers: map[object.SearchMatchType]matcher{
@ -311,3 +324,10 @@ func WithMaxBatchDelay(d time.Duration) Option {
}
}
}
// WithEpochState return option to specify a source of current epoch height.
func WithEpochState(s EpochState) Option {
return func(c *cfg) {
c.epochState = s
}
}

View file

@ -1,6 +1,7 @@
package meta_test
import (
"math"
"os"
"testing"
@ -18,6 +19,12 @@ import (
"github.com/stretchr/testify/require"
)
type epochState struct{}
func (s epochState) CurrentEpoch() uint64 {
return math.MaxUint64
}
// saves "big" object in DB.
func putBig(db *meta.DB, obj *object.Object) error {
return metaPut(db, obj, nil)
@ -36,8 +43,13 @@ func testSelect(t *testing.T, db *meta.DB, cnr cid.ID, fs object.SearchFilters,
func newDB(t testing.TB, opts ...meta.Option) *meta.DB {
path := t.Name()
bdb := meta.New(append([]meta.Option{meta.WithPath(path), meta.WithPermissions(0600)},
opts...)...)
bdb := meta.New(
append([]meta.Option{
meta.WithPath(path),
meta.WithPermissions(0600),
meta.WithEpochState(epochState{}),
}, opts...)...,
)
require.NoError(t, bdb.Open(false))
require.NoError(t, bdb.Init())

View file

@ -4,6 +4,7 @@ import (
"encoding/binary"
"errors"
"fmt"
"math"
"path/filepath"
"testing"
@ -11,12 +12,18 @@ import (
"go.etcd.io/bbolt"
)
type epochStateImpl struct{}
func (s epochStateImpl) CurrentEpoch() uint64 {
return math.MaxUint64
}
func TestVersion(t *testing.T) {
dir := t.TempDir()
newDB := func(t *testing.T) *DB {
return New(WithPath(filepath.Join(dir, t.Name())),
WithPermissions(0600))
WithPermissions(0600), WithEpochState(epochStateImpl{}))
}
check := func(t *testing.T, db *DB) {
require.NoError(t, db.boltDB.View(func(tx *bbolt.Tx) error {

View file

@ -1,6 +1,7 @@
package shard
import (
"math"
"os"
"path/filepath"
"testing"
@ -23,6 +24,12 @@ import (
"go.uber.org/zap/zaptest"
)
type epochState struct{}
func (s epochState) CurrentEpoch() uint64 {
return math.MaxUint64
}
func TestShardOpen(t *testing.T) {
dir := t.TempDir()
metaPath := filepath.Join(dir, "meta")
@ -36,7 +43,7 @@ func TestShardOpen(t *testing.T) {
blobstor.WithSmallSizeLimit(1),
blobstor.WithBlobovniczaShallowWidth(1),
blobstor.WithBlobovniczaShallowDepth(1)),
WithMetaBaseOptions(meta.WithPath(metaPath)),
WithMetaBaseOptions(meta.WithPath(metaPath), meta.WithEpochState(epochState{})),
WithPiloramaOptions(
pilorama.WithPath(filepath.Join(dir, "pilorama"))),
WithWriteCache(true),
@ -82,7 +89,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
sh := New(
WithBlobStorOptions(blobOpts...),
WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))),
WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta"))))
WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(epochState{})))
require.NoError(t, sh.Open())
require.NoError(t, sh.Init())
@ -107,7 +114,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
sh = New(
WithBlobStorOptions(blobOpts...),
WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))),
WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta_new"))),
WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta_new")), meta.WithEpochState(epochState{})),
WithRefillMetabase(true))
require.NoError(t, sh.Open())
require.NoError(t, sh.Init())
@ -134,6 +141,7 @@ func TestRefillMetabase(t *testing.T) {
WithBlobStorOptions(blobOpts...),
WithMetaBaseOptions(
meta.WithPath(filepath.Join(p, "meta")),
meta.WithEpochState(epochState{}),
),
WithPiloramaOptions(
pilorama.WithPath(filepath.Join(p, "pilorama"))),
@ -299,6 +307,7 @@ func TestRefillMetabase(t *testing.T) {
WithBlobStorOptions(blobOpts...),
WithMetaBaseOptions(
meta.WithPath(filepath.Join(p, "meta_restored")),
meta.WithEpochState(epochState{}),
),
WithPiloramaOptions(
pilorama.WithPath(filepath.Join(p, "pilorama_another"))),

View file

@ -32,6 +32,7 @@ func TestShard_Lock(t *testing.T) {
),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(rootPath, "meta")),
meta.WithEpochState(epochState{}),
),
shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) {
sh.HandleDeletedLocks(addresses)

View file

@ -2,6 +2,7 @@ package shard_test
import (
"crypto/sha256"
"math"
"math/rand"
"os"
"path/filepath"
@ -25,6 +26,12 @@ import (
"go.uber.org/zap"
)
type epochState struct{}
func (s epochState) CurrentEpoch() uint64 {
return math.MaxUint64
}
func newShard(t testing.TB, enableWriteCache bool) *shard.Shard {
return newCustomShard(t, t.TempDir(), enableWriteCache,
[]writecache.Option{writecache.WithMaxMemSize(0)},
@ -49,6 +56,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts
),
shard.WithMetaBaseOptions(
meta.WithPath(filepath.Join(rootPath, "meta")),
meta.WithEpochState(epochState{}),
),
shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))),
shard.WithWriteCache(enableWriteCache),