2023-12-22 09:58:20 +00:00
|
|
|
package writecache
|
2023-06-22 11:55:30 +00:00
|
|
|
|
|
|
|
import (
|
2023-08-31 16:26:47 +00:00
|
|
|
"context"
|
2024-09-09 15:37:06 +00:00
|
|
|
"fmt"
|
2023-06-22 11:55:30 +00:00
|
|
|
"sync"
|
2024-06-26 08:19:33 +00:00
|
|
|
"sync/atomic"
|
2023-06-22 11:55:30 +00:00
|
|
|
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
|
2024-09-10 08:49:17 +00:00
|
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
2023-06-22 11:55:30 +00:00
|
|
|
"go.uber.org/zap"
|
|
|
|
)
|
|
|
|
|
|
|
|
type cache struct {
|
|
|
|
options
|
|
|
|
|
|
|
|
mode mode.Mode
|
|
|
|
modeMtx sync.RWMutex
|
|
|
|
|
|
|
|
// flushCh is a channel with objects to flush.
|
2023-10-06 07:22:35 +00:00
|
|
|
flushCh chan objectInfo
|
2023-09-19 05:46:19 +00:00
|
|
|
// cancel is cancel function, protected by modeMtx in Close.
|
2024-06-26 08:19:33 +00:00
|
|
|
cancel atomic.Value
|
2023-06-22 11:55:30 +00:00
|
|
|
// wg is a wait group for flush workers.
|
|
|
|
wg sync.WaitGroup
|
|
|
|
// fsTree contains big files stored directly on file-system.
|
|
|
|
fsTree *fstree.FSTree
|
2024-09-10 08:01:30 +00:00
|
|
|
// counter contains atomic counters for the number of objects stored in cache.
|
|
|
|
counter *fstree.SimpleCounter
|
2023-06-22 11:55:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// wcStorageType is used for write-cache operations logging.
|
|
|
|
const wcStorageType = "write-cache"
|
|
|
|
|
|
|
|
type objectInfo struct {
|
2024-09-10 08:49:17 +00:00
|
|
|
addr oid.Address
|
|
|
|
size uint64
|
2023-06-22 11:55:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
const (
|
2024-09-10 10:12:17 +00:00
|
|
|
defaultMaxObjectSize = 64 * 1024 * 1024 // 64 MiB
|
|
|
|
defaultMaxCacheSize = 1 << 30 // 1 GiB
|
2023-06-22 11:55:30 +00:00
|
|
|
)
|
|
|
|
|
2024-09-11 14:08:52 +00:00
|
|
|
var dummyCanceler context.CancelFunc = func() {}
|
2023-06-22 11:55:30 +00:00
|
|
|
|
|
|
|
// New creates new writecache instance.
|
2023-12-22 09:58:20 +00:00
|
|
|
func New(opts ...Option) Cache {
|
2023-06-22 11:55:30 +00:00
|
|
|
c := &cache{
|
2023-10-06 07:22:35 +00:00
|
|
|
flushCh: make(chan objectInfo),
|
2024-06-10 14:48:05 +00:00
|
|
|
mode: mode.Disabled,
|
2024-09-10 08:01:30 +00:00
|
|
|
counter: fstree.NewSimpleCounter(),
|
2023-06-22 11:55:30 +00:00
|
|
|
|
|
|
|
options: options{
|
2024-10-21 07:22:54 +00:00
|
|
|
log: logger.NewLoggerWrapper(zap.NewNop()),
|
2024-09-10 10:12:17 +00:00
|
|
|
maxObjectSize: defaultMaxObjectSize,
|
|
|
|
workersCount: defaultFlushWorkersCount,
|
|
|
|
maxCacheSize: defaultMaxCacheSize,
|
|
|
|
metrics: DefaultMetrics(),
|
|
|
|
flushSizeLimit: defaultFlushWorkersCount * defaultMaxObjectSize,
|
2023-06-22 11:55:30 +00:00
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
for i := range opts {
|
|
|
|
opts[i](&c.options)
|
|
|
|
}
|
|
|
|
|
|
|
|
return c
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetLogger sets logger. It is used after the shard ID was generated to use it in logs.
|
|
|
|
func (c *cache) SetLogger(l *logger.Logger) {
|
|
|
|
c.log = l
|
|
|
|
}
|
|
|
|
|
2023-12-22 09:58:20 +00:00
|
|
|
func (c *cache) DumpInfo() Info {
|
|
|
|
return Info{
|
2023-06-22 11:55:30 +00:00
|
|
|
Path: c.path,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Open opens and initializes database. Reads object counters from the ObjectCounters instance.
|
2024-06-04 13:28:47 +00:00
|
|
|
func (c *cache) Open(_ context.Context, mod mode.Mode) error {
|
2024-02-09 06:17:17 +00:00
|
|
|
c.modeMtx.Lock()
|
|
|
|
defer c.modeMtx.Unlock()
|
2024-06-04 13:28:47 +00:00
|
|
|
c.mode = mod
|
|
|
|
if mod.NoMetabase() {
|
2024-02-09 06:17:17 +00:00
|
|
|
return nil
|
|
|
|
}
|
2024-06-04 13:28:47 +00:00
|
|
|
err := c.openStore(mode.ConvertToComponentModeDegraded(mod))
|
2023-06-22 11:55:30 +00:00
|
|
|
if err != nil {
|
|
|
|
return metaerr.Wrap(err)
|
|
|
|
}
|
2023-08-11 08:32:43 +00:00
|
|
|
return metaerr.Wrap(c.initCounters())
|
2023-06-22 11:55:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Init runs necessary services.
|
2024-10-21 13:27:28 +00:00
|
|
|
func (c *cache) Init(ctx context.Context) error {
|
2024-06-04 13:28:47 +00:00
|
|
|
c.metrics.SetMode(mode.ConvertToComponentModeDegraded(c.mode))
|
2024-10-21 13:27:28 +00:00
|
|
|
if err := c.flushAndDropBBoltDB(ctx); err != nil {
|
2024-09-09 15:37:06 +00:00
|
|
|
return fmt.Errorf("flush previous version write-cache database: %w", err)
|
|
|
|
}
|
2024-10-21 13:27:28 +00:00
|
|
|
ctx, cancel := context.WithCancel(context.WithoutCancel(ctx)) // canceling performed by cache
|
2024-06-26 08:19:33 +00:00
|
|
|
c.cancel.Store(cancel)
|
2023-09-19 05:46:19 +00:00
|
|
|
c.runFlushLoop(ctx)
|
2023-06-22 11:55:30 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op.
|
2024-10-21 13:27:28 +00:00
|
|
|
func (c *cache) Close(ctx context.Context) error {
|
2024-06-26 08:19:33 +00:00
|
|
|
if cancelValue := c.cancel.Swap(dummyCanceler); cancelValue != nil {
|
|
|
|
cancelValue.(context.CancelFunc)()
|
|
|
|
}
|
2023-06-22 11:55:30 +00:00
|
|
|
// We cannot lock mutex for the whole operation duration
|
|
|
|
// because it is taken by some background workers, so `wg.Wait()` is done without modeMtx.
|
|
|
|
c.modeMtx.Lock()
|
|
|
|
c.mode = mode.DegradedReadOnly // prevent new operations from being processed
|
|
|
|
c.modeMtx.Unlock()
|
|
|
|
|
|
|
|
c.wg.Wait()
|
|
|
|
|
|
|
|
c.modeMtx.Lock()
|
|
|
|
defer c.modeMtx.Unlock()
|
|
|
|
|
|
|
|
var err error
|
2024-09-09 15:37:06 +00:00
|
|
|
if c.fsTree != nil {
|
2024-10-21 13:27:28 +00:00
|
|
|
err = c.fsTree.Close(ctx)
|
2023-06-22 11:55:30 +00:00
|
|
|
if err != nil {
|
2024-09-09 15:37:06 +00:00
|
|
|
c.fsTree = nil
|
2023-06-22 11:55:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
c.metrics.Close()
|
|
|
|
return nil
|
|
|
|
}
|
2024-02-13 06:09:47 +00:00
|
|
|
|
|
|
|
func (c *cache) GetMetrics() Metrics {
|
|
|
|
return c.metrics
|
|
|
|
}
|