[#970] fstree: Move file locking to the generic writer

It is not a part of FSTree itself, but rather a way to solve concurrent
counter update on non-linux implementations. New linux implementations
is pretty simple: link fails when the file exists, unlink fails when the
file doesn't exist.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Evgenii Stratonikov 2024-02-08 18:04:18 +03:00 committed by Evgenii Stratonikov
parent fb74524ac7
commit abd502215f
4 changed files with 37 additions and 25 deletions

View file

@ -18,6 +18,11 @@ func (c *noopCounter) Set(uint64) {}
func (c *noopCounter) Inc() {} func (c *noopCounter) Inc() {}
func (c *noopCounter) Dec() {} func (c *noopCounter) Dec() {}
func counterEnabled(c FileCounter) bool {
_, noop := c.(*noopCounter)
return !noop
}
type SimpleCounter struct { type SimpleCounter struct {
v atomic.Uint64 v atomic.Uint64
} }

View file

@ -53,9 +53,7 @@ type FSTree struct {
readOnly bool readOnly bool
metrics Metrics metrics Metrics
fileGuard keyLock
fileCounter FileCounter fileCounter FileCounter
fileCounterEnabled bool
writer writer writer writer
} }
@ -88,14 +86,13 @@ func New(opts ...Option) *FSTree {
Depth: 4, Depth: 4,
DirNameLen: DirNameLen, DirNameLen: DirNameLen,
metrics: &noopMetrics{}, metrics: &noopMetrics{},
fileGuard: &noopKeyLock{},
fileCounter: &noopCounter{}, fileCounter: &noopCounter{},
log: &logger.Logger{Logger: zap.L()}, log: &logger.Logger{Logger: zap.L()},
} }
for i := range opts { for i := range opts {
opts[i](f) opts[i](f)
} }
f.writer = newGenericWriteData(f) f.writer = newGenericWriteData(f.fileCounter, f.Permissions, f.noSync)
return f return f
} }
@ -444,7 +441,7 @@ func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.G
// initFileCounter walks the file tree rooted at FSTree's root, // initFileCounter walks the file tree rooted at FSTree's root,
// counts total items count, inits counter and returns number of stored objects. // counts total items count, inits counter and returns number of stored objects.
func (t *FSTree) initFileCounter() error { func (t *FSTree) initFileCounter() error {
if !t.fileCounterEnabled { if !counterEnabled(t.fileCounter) {
return nil return nil
} }

View file

@ -10,6 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
) )
@ -22,19 +23,31 @@ type genericWriter struct {
perm fs.FileMode perm fs.FileMode
flags int flags int
t *FSTree fileGuard keyLock
fileCounter FileCounter
fileCounterEnabled bool
suffix atomic.Uint64 suffix atomic.Uint64
} }
func newGenericWriteData(t *FSTree) writer { func newGenericWriteData(c FileCounter, perm fs.FileMode, noSync bool) writer {
flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL
if !t.noSync { if !noSync {
flags |= os.O_SYNC flags |= os.O_SYNC
} }
var fileGuard keyLock = &noopKeyLock{}
fileCounterEnabled := counterEnabled(c)
if fileCounterEnabled {
fileGuard = utilSync.NewKeyLocker[string]()
}
var w = &genericWriter{ var w = &genericWriter{
perm: t.Permissions, perm: perm,
flags: flags, flags: flags,
t: t,
fileCounterEnabled: fileCounterEnabled,
fileGuard: fileGuard,
fileCounter: c,
} }
return w return w
} }
@ -46,9 +59,9 @@ func (w *genericWriter) writeData(p string, data []byte) error {
// writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p.
func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error { func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error {
if w.t.fileCounterEnabled { if w.fileCounterEnabled {
w.t.fileGuard.Lock(p) w.fileGuard.Lock(p)
defer w.t.fileGuard.Unlock(p) defer w.fileGuard.Unlock(p)
} }
err := w.writeFile(tmpPath, data) err := w.writeFile(tmpPath, data)
@ -64,15 +77,15 @@ func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error {
return err return err
} }
if w.t.fileCounterEnabled { if w.fileCounterEnabled {
w.t.fileCounter.Inc() w.fileCounter.Inc()
var targetFileExists bool var targetFileExists bool
if _, e := os.Stat(p); e == nil { if _, e := os.Stat(p); e == nil {
targetFileExists = true targetFileExists = true
} }
err = os.Rename(tmpPath, p) err = os.Rename(tmpPath, p)
if err == nil && targetFileExists { if err == nil && targetFileExists {
w.t.fileCounter.Dec() w.fileCounter.Dec()
} }
} else { } else {
err = os.Rename(tmpPath, p) err = os.Rename(tmpPath, p)
@ -96,12 +109,12 @@ func (w *genericWriter) writeFile(p string, data []byte) error {
func (w *genericWriter) removeFile(p string) error { func (w *genericWriter) removeFile(p string) error {
var err error var err error
if w.t.fileCounterEnabled { if w.fileCounterEnabled {
w.t.fileGuard.Lock(p) w.fileGuard.Lock(p)
err = os.Remove(p) err = os.Remove(p)
w.t.fileGuard.Unlock(p) w.fileGuard.Unlock(p)
if err == nil { if err == nil {
w.t.fileCounter.Dec() w.fileCounter.Dec()
} }
} else { } else {
err = os.Remove(p) err = os.Remove(p)

View file

@ -4,7 +4,6 @@ import (
"io/fs" "io/fs"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -48,9 +47,7 @@ func WithMetrics(m Metrics) Option {
func WithFileCounter(c FileCounter) Option { func WithFileCounter(c FileCounter) Option {
return func(f *FSTree) { return func(f *FSTree) {
f.fileCounterEnabled = true
f.fileCounter = c f.fileCounter = c
f.fileGuard = utilSync.NewKeyLocker[string]()
} }
} }