TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 101 Pull requests 6 Projects Releases 1 Activity Actions

fstree: Use O_TMPFILE for temporary files #970

Merged
fyrchik merged 7 commits from fyrchik/frostfs-node:fix-fstree into master 2024-02-09 16:12:13 +00:00
Conversation 3 Commits 7 Files changed 9 +97 -37
6 changed files with 97 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 8b7d50708b - Show all commits

7
pkg/local_object_storage/blobstor/fstree/control.go
View file

@ -16,6 +16,13 @@ func (t *FSTree) Init() error {
if err := util.MkdirAllX(t.RootPath, t.Permissions); err != nil {
return err
}
if !t.readOnly {
f := newSpecificWriteData(t.fileCounter, t.RootPath, t.Permissions, t.noSync)
if f != nil {
t.writeData = f
}
}
return t.initFileCounter()
}

4
pkg/local_object_storage/blobstor/fstree/fstree.go
View file

@ -10,7 +10,6 @@ import (
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
@ -58,7 +57,7 @@ type FSTree struct {
fileCounter FileCounter
fileCounterEnabled bool
suffix atomic.Uint64
writeData func(string, []byte) error
}
// Info groups the information about file storage.
@ -96,6 +95,7 @@ func New(opts ...Option) *FSTree {
for i := range opts {
opts[i](f)
}
f.writeData = newGenericWriteData(f)
return f
}

58
pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go
View file

@ -1,5 +1,3 @@
//go:build !linux
package fstree
import (
@ -7,24 +5,46 @@ import (
"io/fs"
"os"
"strconv"
"sync/atomic"
"syscall"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
)
func (t *FSTree) writeData(p string, data []byte) error {
tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10)
return t.writeAndRename(tmpPath, p, data)
type genericWriter struct {
perm fs.FileMode
flags int
t *FSTree
suffix atomic.Uint64
}
func newGenericWriteData(t *FSTree) func(string, []byte) error {
flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL
if !t.noSync {
flags |= os.O_SYNC
}
var w = &genericWriter{
perm: t.Permissions,
flags: flags,
t: t,
}
return w.writeData
}
func (w *genericWriter) writeData(p string, data []byte) error {
tmpPath := p + "#" + strconv.FormatUint(w.suffix.Add(1), 10)
return w.writeAndRename(tmpPath, p, data)
}
// writeAndRename opens tmpPath exclusively, writes data to it and renames it to p.
func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error {
if t.fileCounterEnabled {
t.fileGuard.Lock(p)
defer t.fileGuard.Unlock(p)
func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error {
if w.t.fileCounterEnabled {
w.t.fileGuard.Lock(p)
defer w.t.fileGuard.Unlock(p)
}
err := t.writeFile(tmpPath, data)
err := w.writeFile(tmpPath, data)
if err != nil {
var pe *fs.PathError
if errors.As(err, &pe) {
@ -37,15 +57,15 @@ func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error {
return err
}
if t.fileCounterEnabled {
t.fileCounter.Inc()
if w.t.fileCounterEnabled {
w.t.fileCounter.Inc()
var targetFileExists bool
if _, e := os.Stat(p); e == nil {
targetFileExists = true
}
err = os.Rename(tmpPath, p)
if err == nil && targetFileExists {
t.fileCounter.Dec()
w.t.fileCounter.Dec()
}
} else {
err = os.Rename(tmpPath, p)
@ -53,18 +73,10 @@ func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error {
return err
}
func (t *FSTree) writeFlags() int {
flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL
if t.noSync {
return flags
}
return flags | os.O_SYNC
}
// writeFile writes data to a file with path p.
// The code is copied from `os.WriteFile` with minor corrections for flags.
func (t *FSTree) writeFile(p string, data []byte) error {
f, err := os.OpenFile(p, t.writeFlags(), t.Permissions)
func (w *genericWriter) writeFile(p string, data []byte) error {
f, err := os.OpenFile(p, w.flags, w.perm)
if err != nil {
return err
}

46
pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go
View file

@ -4,26 +4,52 @@ package fstree
import (
"errors"
"io/fs"
"strconv"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"golang.org/x/sys/unix"
)
func (t *FSTree) writeData(p string, data []byte) error {
err := t.writeFile(p, data)
type linuxWriter struct {
root string
perm uint32
flags int
counter FileCounter
}
func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync bool) func(string, []byte) error {
flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC
if !noSync {
flags |= unix.O_DSYNC
}
fd, err := unix.Open(root, flags, uint32(perm))
if err != nil {
// Which means that OS-specific writeData can't be created
// and FSTree should use the generic one.
return nil
}
_ = unix.Close(fd) // Don't care about error.
w := &linuxWriter{
root: root,
perm: uint32(perm),
flags: flags,
counter: c,
}
return w.writeData
}
func (w *linuxWriter) writeData(p string, data []byte) error {
err := w.writeFile(p, data)
if errors.Is(err, unix.ENOSPC) {
return common.ErrNoSpace
}
return err
}
func (t *FSTree) writeFile(p string, data []byte) error {
flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC
if !t.noSync {
flags |= unix.O_DSYNC
}
fd, err := unix.Open(t.RootPath, flags, uint32(t.Permissions))
func (w *linuxWriter) writeFile(p string, data []byte) error {
fd, err := unix.Open(w.root, w.flags, w.perm)
if err != nil {
return err
}
@ -32,8 +58,10 @@ func (t *FSTree) writeFile(p string, data []byte) error {
if err == nil {
if n == len(data) {
err = unix.Linkat(unix.AT_FDCWD, tmpPath, unix.AT_FDCWD, p, unix.AT_SYMLINK_FOLLOW)
if err == nil {
w.counter.Inc()
}
if errors.Is(err, unix.EEXIST) {
// https://github.com/nspcc-dev/neofs-node/issues/2563
err = nil
}
} else {

9
pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go Normal file
View file

@ -0,0 +1,9 @@
//go:build !linux
package fstree
import "io/fs"
func newSpecificWriteData(_ FileCounter, _ string, _ fs.FileMode, _ bool) func(string, []byte) error {
return nil
}

10
pkg/local_object_storage/shard/control_test.go
View file

@ -2,6 +2,7 @@ package shard
import (
"context"
"fmt"
"io/fs"
"math"
"os"
@ -11,7 +12,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
@ -145,8 +145,12 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
require.NoError(t, sh.Close())
addr := object.AddressOf(obj)
_, err = fsTree.Put(context.Background(), common.PutPrm{Address: addr, RawData: []byte("not an object")})
require.NoError(t, err)
// This is copied from `fstree.treePath()` to avoid exporting function just for tests.
{
saddr := addr.Object().EncodeToString() + "." + addr.Container().EncodeToString()
p := fmt.Sprintf("%s/%s/%s", fsTree.RootPath, saddr[:2], saddr[2:])
require.NoError(t, os.WriteFile(p, []byte("not an object"), fsTree.Permissions))
}
sh = New(
WithID(NewIDFromBytes([]byte{})),