ale64bit/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#68] node: Add a local storage implementation based on bbolt thin wrapper

Browse source 
Signed-off-by: Alejandro Lopez <a.lopez@yadro.com>
This commit is contained in:
Alejandro Lopez 2023-02-28 18:25:34 +03:00
parent 79ba34714a
commit 0c9f360a3c
22 changed files with 600 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
cmd/frostfs-node/config.go
View file

@ -31,6 +31,7 @@ import (
netmapCore "github.com/TrueCloudLab/frostfs-node/pkg/core/netmap"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/boltstore"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine"
meta "github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
@ -687,16 +688,30 @@ func (c *cfg) shardOpts() []shardOptsWithID {
for _, sRead := range shCfg.subStorages {
switch sRead.typ {
case blobovniczatree.Type:
ss = append(ss, blobstor.SubStorage{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(sRead.path),
blobovniczatree.WithPermissions(sRead.perm),
blobovniczatree.WithBlobovniczaSize(sRead.size),
blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth),
blobovniczatree.WithBlobovniczaShallowWidth(sRead.width),
blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize),
// TODO(ale64bit): hardcoded only for testing purposes.
fallthrough
/*
ss = append(ss, blobstor.SubStorage{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(sRead.path),
blobovniczatree.WithPermissions(sRead.perm),
blobovniczatree.WithBlobovniczaSize(sRead.size),
blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth),
blobovniczatree.WithBlobovniczaShallowWidth(sRead.width),
blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize),
blobovniczatree.WithLogger(c.log)),
blobovniczatree.WithLogger(c.log)),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) < shCfg.smallSizeObjectLimit
},
})
*/
case boltstore.Type:
ss = append(ss, blobstor.SubStorage{
Storage: boltstore.New(
boltstore.WithRootPath(sRead.path),
boltstore.WithMode(sRead.perm),
boltstore.WithLogger(c.log)),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) < shCfg.smallSizeObjectLimit
},

14
pkg/local_object_storage/blobstor/blobstor_test.go
View file

@ -6,7 +6,7 @@ import (
"testing"
"github.com/TrueCloudLab/frostfs-node/pkg/core/object"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/boltstore"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
objectSDK "github.com/TrueCloudLab/frostfs-sdk-go/object"
@ -18,9 +18,9 @@ const blobovniczaDir = "blobovniczas"
func defaultStorages(p string, smallSizeLimit uint64) []SubStorage {
return []SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(p, "blobovniczas")),
blobovniczatree.WithBlobovniczaShallowWidth(1)), // default width is 16, slow init
Storage: boltstore.New(
boltstore.WithRootPath(filepath.Join(p, "boltstore")),
),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) <= smallSizeLimit
},
@ -111,9 +111,9 @@ func TestBlobstor_needsCompression(t *testing.T) {
WithUncompressableContentTypes(ct),
WithStorages([]SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(dir, "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowWidth(1)), // default width is 16, slow init
Storage: boltstore.New(
boltstore.WithRootPath(filepath.Join(dir, "boltstore")),
),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) < smallSizeLimit
},

39
pkg/local_object_storage/blobstor/boltstore/boltstore.go Normal file
View file

@ -0,0 +1,39 @@
// boltstore is a thin bbolt-backed common.Storage implementation.
package boltstore
import (
"go.uber.org/atomic"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"go.etcd.io/bbolt"
)
// Type is boltstore storage type used in logs and configuration.
const Type = "boltstore"
type boltStoreImpl struct {
*cfg
size atomic.Uint64
boltDB *bbolt.DB
}
func New(opts ...Option) common.Storage {
store := &boltStoreImpl{
cfg: defaultConfig(),
}
for _, opt := range opts {
opt(store.cfg)
}
return store
}
func (s *boltStoreImpl) Type() string { return Type }
func (s *boltStoreImpl) Path() string { return s.rootPath }
func (s *boltStoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc }
func (s *boltStoreImpl) SetReportErrorFunc(f func(string, error)) { s.reportError = f }
func (s *boltStoreImpl) isFull() bool { return s.size.Load() > s.fullSizeLimit }

69
pkg/local_object_storage/blobstor/boltstore/boltstore_test.go Normal file
View file

@ -0,0 +1,69 @@
package boltstore
import (
"path/filepath"
"testing"
"github.com/TrueCloudLab/frostfs-node/pkg/core/object"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
"github.com/TrueCloudLab/frostfs-node/pkg/util/logger"
objectSDK "github.com/TrueCloudLab/frostfs-sdk-go/object"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
func TestSimpleLifecycle(t *testing.T) {
s := New(
WithRootPath(filepath.Join(t.TempDir(), "boltstore")),
WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
)
t.Cleanup(func() { _ = s.Close() })
require.NoError(t, s.Open(false))
require.NoError(t, s.Init())
obj := blobstortest.NewObject(1024)
addr := object.AddressOf(obj)
d, err := obj.Marshal()
require.NoError(t, err)
{
_, err := s.Put(common.PutPrm{Address: addr, RawData: d, DontCompress: true})
require.NoError(t, err)
}
{
resp, err := s.Exists(common.ExistsPrm{Address: addr})
require.NoError(t, err)
require.True(t, resp.Exists)
}
{
resp, err := s.Get(common.GetPrm{Address: addr})
require.NoError(t, err)
require.Equal(t, obj.Payload(), resp.Object.Payload())
}
{
var objRange objectSDK.Range
objRange.SetOffset(256)
objRange.SetLength(512)
resp, err := s.GetRange(common.GetRangePrm{
Address: addr,
Range: objRange,
})
require.NoError(t, err)
require.Equal(t, obj.Payload()[objRange.GetOffset():objRange.GetOffset()+objRange.GetLength()], resp.Data)
}
{
_, err := s.Delete(common.DeletePrm{Address: addr})
require.NoError(t, err)
}
{
resp, err := s.Exists(common.ExistsPrm{Address: addr})
require.NoError(t, err)
require.False(t, resp.Exists)
}
}

75
pkg/local_object_storage/blobstor/boltstore/control.go Normal file
View file

@ -0,0 +1,75 @@
package boltstore
import (
"fmt"
"os"
"path/filepath"
"github.com/TrueCloudLab/frostfs-node/pkg/util"
"go.etcd.io/bbolt"
"go.uber.org/zap"
)
func (s *boltStoreImpl) Open(readOnly bool) (err error) {
s.log.Debug("creating directory for BoltDB",
zap.String("path", s.rootPath),
zap.Bool("ro", readOnly),
)
if !readOnly {
if err = util.MkdirAllX(filepath.Dir(s.rootPath), s.mode); err != nil {
return
}
}
s.log.Debug("opening BoltDB",
zap.String("path", s.rootPath),
zap.Stringer("permissions", s.mode),
)
s.boltOptions.ReadOnly = readOnly
s.boltDB, err = bbolt.Open(s.rootPath, s.mode, s.boltOptions)
return
}
func (s *boltStoreImpl) Init() error {
s.log.Debug("initializing...",
zap.Uint64("object size limit", s.objSizeLimit),
zap.Uint64("storage size limit", s.fullSizeLimit),
)
if size := s.size.Load(); size != 0 {
s.log.Debug("already initialized", zap.Uint64("size", size))
return nil
}
if !s.boltOptions.ReadOnly {
if err := s.boltDB.Update(func(tx *bbolt.Tx) error {
s.log.Debug("creating bucket", zap.String("name", dataBucket))
if _, err := tx.CreateBucketIfNotExists([]byte(dataBucket)); err != nil {
return fmt.Errorf("(%T) could not create data bucket: %w", s, err)
}
return nil
}); err != nil {
return err
}
}
info, err := os.Stat(s.rootPath)
if err != nil {
return fmt.Errorf("can't determine DB size: %w", err)
}
s.size.Store(uint64(info.Size()))
return nil
}
func (s *boltStoreImpl) Close() error {
if s.boltDB != nil {
s.log.Debug("closing BoltDB", zap.String("path", s.rootPath))
return s.boltDB.Close()
}
return nil
}

47
pkg/local_object_storage/blobstor/boltstore/delete.go Normal file
View file

@ -0,0 +1,47 @@
package boltstore
import (
"fmt"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
apistatus "github.com/TrueCloudLab/frostfs-sdk-go/client/status"
"go.etcd.io/bbolt"
)
func (s *boltStoreImpl) Delete(req common.DeletePrm) (common.DeleteRes, error) {
if s.boltOptions.ReadOnly {
return common.DeleteRes{}, common.ErrReadOnly
}
key := []byte(req.Address.EncodeToString())
deleted := false
err := s.boltDB.Update(func(tx *bbolt.Tx) error {
bucket := tx.Bucket([]byte(dataBucket))
if bucket == nil {
// expected to happen:
// - before initialization step (incorrect usage by design)
// - if DB is corrupted (in future this case should be handled)
return logicerr.Wrap(fmt.Errorf("(%T) data bucket does not exist", s))
}
data := bucket.Get(key)
if data == nil {
return nil
}
if err := bucket.Delete(key); err != nil {
return err
}
s.size.Sub(uint64(len(data)))
deleted = true
return nil
})
if err == nil && !deleted {
return common.DeleteRes{}, logicerr.Wrap(apistatus.ObjectNotFound{})
}
return common.DeleteRes{}, err
}

27
pkg/local_object_storage/blobstor/boltstore/exists.go Normal file
View file

@ -0,0 +1,27 @@
package boltstore
import (
"fmt"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
"go.etcd.io/bbolt"
)
func (s *boltStoreImpl) Exists(req common.ExistsPrm) (resp common.ExistsRes, err error) {
key := []byte(req.Address.EncodeToString())
err = s.boltDB.View(func(tx *bbolt.Tx) error {
bucket := tx.Bucket([]byte(dataBucket))
if bucket == nil {
// expected to happen:
// - before initialization step (incorrect usage by design)
// - if DB is corrupted (in future this case should be handled)
return logicerr.Wrap(fmt.Errorf("(%T) data bucket does not exist", s))
}
resp.Exists = bucket.Get(key) != nil
return nil
})
return
}

52
pkg/local_object_storage/blobstor/boltstore/get.go Normal file
View file

@ -0,0 +1,52 @@
package boltstore
import (
"fmt"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
apistatus "github.com/TrueCloudLab/frostfs-sdk-go/client/status"
objectSDK "github.com/TrueCloudLab/frostfs-sdk-go/object"
"github.com/nspcc-dev/neo-go/pkg/util/slice"
"go.etcd.io/bbolt"
)
func (s *boltStoreImpl) Get(req common.GetPrm) (common.GetRes, error) {
key := []byte(req.Address.EncodeToString())
var data []byte
// Retrieve the object data.
err := s.boltDB.View(func(tx *bbolt.Tx) error {
bucket := tx.Bucket([]byte(dataBucket))
if bucket == nil {
// expected to happen:
// - before initialization step (incorrect usage by design)
// - if DB is corrupted (in future this case should be handled)
return logicerr.Wrap(fmt.Errorf("(%T) data bucket does not exist", s))
}
if data = bucket.Get(key); data != nil {
data = slice.Copy(data)
}
return nil
})
if err != nil {
return common.GetRes{}, err
}
if data == nil {
return common.GetRes{}, logicerr.Wrap(apistatus.ObjectNotFound{})
}
// Decompress the data.
if data, err = s.compression.Decompress(data); err != nil {
return common.GetRes{}, logicerr.Wrap(fmt.Errorf("could not decompress object data: %w", err))
}
// Unmarshal the SDK object.
obj := objectSDK.New()
if err := obj.Unmarshal(data); err != nil {
return common.GetRes{}, logicerr.Wrap(fmt.Errorf("could not unmarshal the object: %w", err))
}
return common.GetRes{Object: obj, RawData: data}, nil
}

29
pkg/local_object_storage/blobstor/boltstore/get_range.go Normal file
View file

@ -0,0 +1,29 @@
package boltstore
import (
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
apistatus "github.com/TrueCloudLab/frostfs-sdk-go/client/status"
)
func (s *boltStoreImpl) GetRange(req common.GetRangePrm) (common.GetRangeRes, error) {
getResp, err := s.Get(common.GetPrm{
Address: req.Address,
StorageID: req.StorageID,
})
if err != nil {
return common.GetRangeRes{}, err
}
payload := getResp.Object.Payload()
from := req.Range.GetOffset()
to := from + req.Range.GetLength()
if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to {
return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectOutOfRange{})
}
return common.GetRangeRes{
Data: payload[from:to],
}, nil
}

53
pkg/local_object_storage/blobstor/boltstore/iterate.go Normal file
View file

@ -0,0 +1,53 @@
package boltstore
import (
"fmt"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
"go.etcd.io/bbolt"
)
func (s *boltStoreImpl) Iterate(req common.IteratePrm) (common.IterateRes, error) {
return common.IterateRes{}, s.boltDB.View(func(tx *bbolt.Tx) error {
bucket := tx.Bucket([]byte(dataBucket))
if bucket == nil {
// expected to happen:
// - before initialization step (incorrect usage by design)
// - if DB is corrupted (in future this case should be handled)
return logicerr.Wrap(fmt.Errorf("(%T) data bucket does not exist", s))
}
return bucket.ForEach(func(k, v []byte) error {
var err error
elem := common.IterationElement{
ObjectData: v,
}
if err := elem.Address.DecodeString(string(k)); err != nil {
if req.IgnoreErrors {
return nil
}
return logicerr.Wrap(fmt.Errorf("(%T) decoding address string %q: %v", s, string(k), err))
}
if elem.ObjectData, err = s.compression.Decompress(elem.ObjectData); err != nil {
if req.IgnoreErrors {
if req.ErrorHandler != nil {
return req.ErrorHandler(elem.Address, err)
}
return nil
}
return logicerr.Wrap(fmt.Errorf("(%T) decompressing data for address %q: %v", s, elem.Address.String(), err))
}
switch {
case req.Handler != nil:
return req.Handler(elem)
case req.LazyHandler != nil:
return req.LazyHandler(elem.Address, func() ([]byte, error) { return elem.ObjectData, nil })
default:
if !req.IgnoreErrors {
return logicerr.Wrap(fmt.Errorf("(%T) no Handler or LazyHandler set for IteratePrm", s))
}
}
return nil
})
})
}

78
pkg/local_object_storage/blobstor/boltstore/option.go Normal file
View file

@ -0,0 +1,78 @@
package boltstore
import (
"io/fs"
"time"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"github.com/TrueCloudLab/frostfs-node/pkg/util/logger"
"go.etcd.io/bbolt"
"go.uber.org/zap"
)
const (
defaultMode fs.FileMode = 0700
dataBucket = "obj"
)
type cfg struct {
log *logger.Logger
mode fs.FileMode
rootPath string
compression *compression.Config
reportError func(string, error)
fullSizeLimit uint64
objSizeLimit uint64
boltOptions *bbolt.Options
}
func defaultConfig() *cfg {
return &cfg{
log: &logger.Logger{Logger: zap.L()},
mode: defaultMode,
reportError: func(string, error) {},
fullSizeLimit: 1 << 30, // 1GB
objSizeLimit: 1 << 20, // 1MB
boltOptions: &bbolt.Options{
Timeout: 100 * time.Millisecond,
},
}
}
type Option func(*cfg)
func WithLogger(l *logger.Logger) Option {
return func(c *cfg) {
c.log = l
}
}
func WithMode(p fs.FileMode) Option {
return func(c *cfg) {
c.mode = p
}
}
func WithRootPath(p string) Option {
return func(c *cfg) {
c.rootPath = p
}
}
func WithFullSizeLimit(sz uint64) Option {
return func(c *cfg) {
c.fullSizeLimit = sz
}
}
func WithObjectSizeLimit(sz uint64) Option {
return func(c *cfg) {
c.objSizeLimit = sz
}
}
func WithReadOnly(ro bool) Option {
return func(c *cfg) {
c.boltOptions.ReadOnly = ro
}
}

44
pkg/local_object_storage/blobstor/boltstore/put.go Normal file
View file

@ -0,0 +1,44 @@
package boltstore
import (
"fmt"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
"go.etcd.io/bbolt"
)
func (s *boltStoreImpl) Put(req common.PutPrm) (common.PutRes, error) {
if s.boltOptions.ReadOnly {
return common.PutRes{}, common.ErrReadOnly
}
if s.isFull() {
return common.PutRes{}, common.ErrNoSpace
}
if !req.DontCompress {
req.RawData = s.compression.Compress(req.RawData)
}
sz := uint64(len(req.RawData))
key := []byte(req.Address.EncodeToString())
err := s.boltDB.Batch(func(tx *bbolt.Tx) error {
bucket := tx.Bucket([]byte(dataBucket))
if bucket == nil {
// expected to happen:
// - before initialization step (incorrect usage by design)
// - if DB is corrupted (in future this case should be handled)
return logicerr.Wrap(fmt.Errorf("(%T) data bucket does not exist", s))
}
if err := bucket.Put(key, req.RawData); err != nil {
return logicerr.Wrap(fmt.Errorf("(%T) could not save object in bucket: %w", s, err))
}
s.size.Add(sz)
return nil
})
return common.PutRes{StorageID: []byte(s.rootPath)}, err
}

13
pkg/local_object_storage/blobstor/control.go
View file

@ -20,14 +20,14 @@ func (b *BlobStor) Open(readOnly bool) error {
return nil
}
// ErrInitBlobovniczas is returned when blobovnicza initialization fails.
var ErrInitBlobovniczas = errors.New("failure on blobovnicza initialization stage")
// ErrSubstorageInit occurs when the initialization of a substorage system fails when calling Init().
var ErrSubstorageInit = errors.New("substorage initialization failed")
// Init initializes internal data structures and system resources.
//
// If BlobStor is already initialized, no action is taken.
//
// Returns wrapped ErrInitBlobovniczas on blobovnicza tree's initializaiton failure.
// Returns ErrSubstorageInit if initialization of a substorage system fails.
func (b *BlobStor) Init() error {
b.log.Debug("initializing...")
@ -35,10 +35,9 @@ func (b *BlobStor) Init() error {
return err
}
for i := range b.storage {
err := b.storage[i].Storage.Init()
if err != nil {
return fmt.Errorf("%w: %v", ErrInitBlobovniczas, err)
for _, si := range b.storage {
if err := si.Storage.Init(); err != nil {
return fmt.Errorf("%w: type=%q: %v", ErrSubstorageInit, si.Storage.Type(), err)
}
}
return nil

2
pkg/local_object_storage/blobstor/exists_test.go
View file

@ -51,7 +51,7 @@ func TestExists(t *testing.T) {
require.NoError(t, err)
require.False(t, res.Exists)
t.Run("corrupt direcrory", func(t *testing.T) {
t.Run("corrupt directory", func(t *testing.T) {
var bigDir string
de, err := os.ReadDir(dir)
require.NoError(t, err)

6
pkg/local_object_storage/blobstor/iterate_test.go
View file

@ -73,12 +73,6 @@ func TestIterateObjects(t *testing.T) {
require.Equal(t, v.data, data)
if v.big {
require.True(t, descriptor != nil && len(descriptor) == 0)
} else {
require.NotEmpty(t, descriptor)
}
delete(mObjs, string(data))
return nil

2
pkg/local_object_storage/engine/control.go
View file

@ -92,7 +92,7 @@ func (e *StorageEngine) Init() error {
for res := range errCh {
if res.err != nil {
if errors.Is(res.err, blobstor.ErrInitBlobovniczas) {
if errors.Is(res.err, blobstor.ErrSubstorageInit) {
e.log.Error("could not initialize shard, closing and skipping",
zap.String("id", res.id),
zap.Error(res.err))

10
pkg/local_object_storage/engine/engine_test.go
View file

@ -7,7 +7,7 @@ import (
"testing"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/boltstore"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
@ -103,11 +103,9 @@ func testNewEngineWithShards(shards ...*shard.Shard) *StorageEngine {
func newStorages(root string, smallSize uint64) []blobstor.SubStorage {
return []blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(root, "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(1),
blobovniczatree.WithBlobovniczaShallowWidth(1),
blobovniczatree.WithPermissions(0700)),
Storage: boltstore.New(
boltstore.WithRootPath(filepath.Join(root, "boltstore")),
boltstore.WithMode(0700)),
Policy: func(_ *object.Object, data []byte) bool {
return uint64(len(data)) < smallSize
},

19
pkg/local_object_storage/engine/error_test.go
View file

@ -62,6 +62,9 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32)
}
func TestErrorReporting(t *testing.T) {
excludeDir := map[string]bool{
"boltstore": true,
}
t.Run("ignore errors by default", func(t *testing.T) {
e, dir, id := newEngineWithErrorThreshold(t, "", 0)
@ -81,7 +84,7 @@ func TestErrorReporting(t *testing.T) {
checkShardState(t, e, id[0], 0, mode.ReadWrite)
checkShardState(t, e, id[1], 0, mode.ReadWrite)
corruptSubDir(t, filepath.Join(dir, "0"))
corruptSubDir(t, filepath.Join(dir, "0"), excludeDir)
for i := uint32(1); i < 3; i++ {
_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
@ -111,7 +114,7 @@ func TestErrorReporting(t *testing.T) {
checkShardState(t, e, id[0], 0, mode.ReadWrite)
checkShardState(t, e, id[1], 0, mode.ReadWrite)
corruptSubDir(t, filepath.Join(dir, "0"))
corruptSubDir(t, filepath.Join(dir, "0"), excludeDir)
for i := uint32(1); i < errThreshold; i++ {
_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
@ -137,6 +140,9 @@ func TestErrorReporting(t *testing.T) {
// Issue #1186.
func TestBlobstorFailback(t *testing.T) {
// TODO(ale64bit): this test works only if the second substorage is specifically blobovnicza.
t.Skip()
dir, err := os.MkdirTemp("", "*")
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) })
@ -204,16 +210,13 @@ func checkShardState(t *testing.T, e *StorageEngine, id *shard.ID, errCount uint
require.Equal(t, mode, sh.GetMode())
}
// corruptSubDir makes random directory except "blobovnicza" in blobstor FSTree unreadable.
func corruptSubDir(t *testing.T, dir string) {
// corruptSubDir makes random directory in blobstor FSTree unreadable unless it's in excludeDir.
func corruptSubDir(t *testing.T, dir string, excludeDir map[string]bool) {
de, err := os.ReadDir(dir)
require.NoError(t, err)
// FIXME(@cthulhu-rider): copy-paste of unexported const from blobstor package, see #1407
const dirBlobovnicza = "blobovnicza"
for i := range de {
if de[i].IsDir() && de[i].Name() != dirBlobovnicza {
if de[i].IsDir() && !excludeDir[de[i].Name()] {
require.NoError(t, os.Chmod(filepath.Join(dir, de[i].Name()), 0))
return
}

47
pkg/local_object_storage/shard/dump_test.go
View file

@ -10,9 +10,8 @@ import (
"time"
"github.com/TrueCloudLab/frostfs-node/pkg/core/object"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/boltstore"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
@ -20,7 +19,6 @@ import (
"github.com/TrueCloudLab/frostfs-node/pkg/util/logger"
cidtest "github.com/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "github.com/TrueCloudLab/frostfs-sdk-go/object"
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
objecttest "github.com/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/klauspost/compress/zstd"
"github.com/stretchr/testify/require"
@ -41,8 +39,8 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) {
const (
wcSmallObjectSize = 1024 // 1 KiB, goes to write-cache memory
wcBigObjectSize = 4 * 1024 // 4 KiB, goes to write-cache FSTree
bsSmallObjectSize = 10 * 1024 // 10 KiB, goes to blobovnicza DB
bsBigObjectSize = 1024*1024 + 1 // > 1 MiB, goes to blobovnicza FSTree
bsSmallObjectSize = 10 * 1024 // 10 KiB, goes to boltstore
bsBigObjectSize = 1024*1024 + 1 // > 1 MiB, goes to FSTree
)
var sh *shard.Shard
@ -285,7 +283,7 @@ func TestDumpIgnoreErrors(t *testing.T) {
const (
wcSmallObjectSize = 512 // goes to write-cache memory
wcBigObjectSize = wcSmallObjectSize << 1 // goes to write-cache FSTree
bsSmallObjectSize = wcSmallObjectSize << 2 // goes to blobovnicza DB
bsSmallObjectSize = wcSmallObjectSize << 2 // goes to boltstore DB
objCount = 10
headerSize = 400
@ -298,11 +296,8 @@ func TestDumpIgnoreErrors(t *testing.T) {
blobstor.WithCompressObjects(true),
blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(bsPath, "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(1),
blobovniczatree.WithBlobovniczaShallowWidth(sw),
blobovniczatree.WithOpenedCacheSize(1)),
Storage: boltstore.New(
boltstore.WithRootPath(filepath.Join(bsPath, "boltstore"))),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return len(data) < bsSmallObjectSize
},
@ -347,7 +342,7 @@ func TestDumpIgnoreErrors(t *testing.T) {
}
// There are 3 different types of errors to consider.
// To setup envirionment we use implementation details so this test must be updated
// To setup environment we use implementation details so this test must be updated
// if any of them are changed.
{
// 1. Invalid object in fs tree.
@ -373,22 +368,24 @@ func TestDumpIgnoreErrors(t *testing.T) {
require.NoError(t, sh.SetMode(mode.ReadOnly))
{
// TODO(ale64bit): these checks are specific to the substorage implementation.
// 2. Invalid object in blobovnicza.
// 2.1. Invalid blobovnicza.
bTree := filepath.Join(bsPath, "blobovnicza")
data := make([]byte, 1024)
rand.Read(data)
require.NoError(t, os.WriteFile(filepath.Join(bTree, "0", "2"), data, 0))
// bTree := filepath.Join(bsPath, "blobovnicza")
// data := make([]byte, 1024)
// rand.Read(data)
// require.NoError(t, os.WriteFile(filepath.Join(bTree, "0", "2"), data, 0))
// 2.2. Invalid object in valid blobovnicza.
var prm blobovnicza.PutPrm
prm.SetAddress(oid.Address{})
prm.SetMarshaledObject(corruptedData)
b := blobovnicza.New(blobovnicza.WithPath(filepath.Join(bTree, "1", "2")))
require.NoError(t, b.Open())
_, err := b.Put(prm)
require.NoError(t, err)
require.NoError(t, b.Close())
// // 2.2. Invalid object in valid blobovnicza.
// var prm blobovnicza.PutPrm
// prm.SetAddress(oid.Address{})
// prm.SetMarshaledObject(corruptedData)
// b := blobovnicza.New(blobovnicza.WithPath(filepath.Join(bTree, "1", "2")))
// require.NoError(t, b.Open())
// _, err := b.Put(prm)
// require.NoError(t, err)
// require.NoError(t, b.Close())
}
{

8
pkg/local_object_storage/shard/lock_test.go
View file

@ -7,7 +7,7 @@ import (
objectcore "github.com/TrueCloudLab/frostfs-node/pkg/core/object"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/boltstore"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
@ -30,10 +30,8 @@ func TestShard_Lock(t *testing.T) {
shard.WithBlobStorOptions(
blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(2),
blobovniczatree.WithBlobovniczaShallowWidth(2)),
Storage: boltstore.New(
boltstore.WithRootPath(filepath.Join(rootPath, "blob", "boltstore"))),
Policy: func(_ *object.Object, data []byte) bool {
return len(data) <= 1<<20
},

10
pkg/local_object_storage/shard/range_test.go
View file

@ -7,7 +7,7 @@ import (
"github.com/TrueCloudLab/frostfs-node/pkg/core/object"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/boltstore"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
@ -69,11 +69,9 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) {
[]writecache.Option{writecache.WithMaxObjectSize(writeCacheMaxSize)},
[]blobstor.Option{blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(1),
blobovniczatree.WithBlobovniczaShallowWidth(1)),
Storage: boltstore.New(
boltstore.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
boltstore.WithRootPath(filepath.Join(t.TempDir(), "blob", "boltstore"))),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return len(data) <= smallObjectSize
},

10
pkg/local_object_storage/shard/shard_test.go
View file

@ -7,7 +7,7 @@ import (
"testing"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/boltstore"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"github.com/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
@ -51,11 +51,9 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts
blobstor.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
blobstor.WithStorages([]blobstor.SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
blobovniczatree.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")),
blobovniczatree.WithBlobovniczaShallowDepth(1),
blobovniczatree.WithBlobovniczaShallowWidth(1)),
Storage: boltstore.New(
boltstore.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
boltstore.WithRootPath(filepath.Join(rootPath, "blob", "boltstore"))),
Policy: func(_ *object.Object, data []byte) bool {
return len(data) <= 1<<20
},