[#970] fstree: Handle unsupported O_TMPFILE

Metabase test relied on this behaviour, so fix the test too.

Cherry-picking was hard and did too many conflicts,
here is an original PR:
https://github.com/nspcc-dev/neofs-node/pull/2624

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Evgenii Stratonikov 2024-02-08 11:57:23 +03:00 committed by Evgenii Stratonikov
parent fc31b9c947
commit 7f692409cf
6 changed files with 97 additions and 37 deletions

View file

@ -16,6 +16,13 @@ func (t *FSTree) Init() error {
if err := util.MkdirAllX(t.RootPath, t.Permissions); err != nil { if err := util.MkdirAllX(t.RootPath, t.Permissions); err != nil {
return err return err
} }
if !t.readOnly {
f := newSpecificWriteData(t.fileCounter, t.RootPath, t.Permissions, t.noSync)
if f != nil {
t.writeData = f
}
}
return t.initFileCounter() return t.initFileCounter()
} }

View file

@ -10,7 +10,6 @@ import (
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
"sync/atomic"
"syscall" "syscall"
"time" "time"
@ -58,7 +57,7 @@ type FSTree struct {
fileCounter FileCounter fileCounter FileCounter
fileCounterEnabled bool fileCounterEnabled bool
suffix atomic.Uint64 writeData func(string, []byte) error
} }
// Info groups the information about file storage. // Info groups the information about file storage.
@ -96,6 +95,7 @@ func New(opts ...Option) *FSTree {
for i := range opts { for i := range opts {
opts[i](f) opts[i](f)
} }
f.writeData = newGenericWriteData(f)
return f return f
} }

View file

@ -1,5 +1,3 @@
//go:build !linux
package fstree package fstree
import ( import (
@ -7,24 +5,46 @@ import (
"io/fs" "io/fs"
"os" "os"
"strconv" "strconv"
"sync/atomic"
"syscall" "syscall"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
) )
func (t *FSTree) writeData(p string, data []byte) error { type genericWriter struct {
tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) perm fs.FileMode
return t.writeAndRename(tmpPath, p, data) flags int
t *FSTree
suffix atomic.Uint64
}
func newGenericWriteData(t *FSTree) func(string, []byte) error {
flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL
if !t.noSync {
flags |= os.O_SYNC
}
var w = &genericWriter{
perm: t.Permissions,
flags: flags,
t: t,
}
return w.writeData
}
func (w *genericWriter) writeData(p string, data []byte) error {
tmpPath := p + "#" + strconv.FormatUint(w.suffix.Add(1), 10)
return w.writeAndRename(tmpPath, p, data)
} }
// writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p.
func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error {
if t.fileCounterEnabled { if w.t.fileCounterEnabled {
t.fileGuard.Lock(p) w.t.fileGuard.Lock(p)
defer t.fileGuard.Unlock(p) defer w.t.fileGuard.Unlock(p)
} }
err := t.writeFile(tmpPath, data) err := w.writeFile(tmpPath, data)
if err != nil { if err != nil {
var pe *fs.PathError var pe *fs.PathError
if errors.As(err, &pe) { if errors.As(err, &pe) {
@ -37,15 +57,15 @@ func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error {
return err return err
} }
if t.fileCounterEnabled { if w.t.fileCounterEnabled {
t.fileCounter.Inc() w.t.fileCounter.Inc()
var targetFileExists bool var targetFileExists bool
if _, e := os.Stat(p); e == nil { if _, e := os.Stat(p); e == nil {
targetFileExists = true targetFileExists = true
} }
err = os.Rename(tmpPath, p) err = os.Rename(tmpPath, p)
if err == nil && targetFileExists { if err == nil && targetFileExists {
t.fileCounter.Dec() w.t.fileCounter.Dec()
} }
} else { } else {
err = os.Rename(tmpPath, p) err = os.Rename(tmpPath, p)
@ -53,18 +73,10 @@ func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error {
return err return err
} }
func (t *FSTree) writeFlags() int {
flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL
if t.noSync {
return flags
}
return flags | os.O_SYNC
}
// writeFile writes data to a file with path p. // writeFile writes data to a file with path p.
// The code is copied from `os.WriteFile` with minor corrections for flags. // The code is copied from `os.WriteFile` with minor corrections for flags.
func (t *FSTree) writeFile(p string, data []byte) error { func (w *genericWriter) writeFile(p string, data []byte) error {
f, err := os.OpenFile(p, t.writeFlags(), t.Permissions) f, err := os.OpenFile(p, w.flags, w.perm)
if err != nil { if err != nil {
return err return err
} }

View file

@ -4,26 +4,52 @@ package fstree
import ( import (
"errors" "errors"
"io/fs"
"strconv" "strconv"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
func (t *FSTree) writeData(p string, data []byte) error { type linuxWriter struct {
err := t.writeFile(p, data) root string
perm uint32
flags int
counter FileCounter
}
func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync bool) func(string, []byte) error {
flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC
if !noSync {
flags |= unix.O_DSYNC
}
fd, err := unix.Open(root, flags, uint32(perm))
if err != nil {
// Which means that OS-specific writeData can't be created
// and FSTree should use the generic one.
return nil
}
_ = unix.Close(fd) // Don't care about error.
w := &linuxWriter{
root: root,
perm: uint32(perm),
flags: flags,
counter: c,
}
return w.writeData
}
func (w *linuxWriter) writeData(p string, data []byte) error {
err := w.writeFile(p, data)
if errors.Is(err, unix.ENOSPC) { if errors.Is(err, unix.ENOSPC) {
return common.ErrNoSpace return common.ErrNoSpace
} }
return err return err
} }
func (t *FSTree) writeFile(p string, data []byte) error { func (w *linuxWriter) writeFile(p string, data []byte) error {
flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC fd, err := unix.Open(w.root, w.flags, w.perm)
if !t.noSync {
flags |= unix.O_DSYNC
}
fd, err := unix.Open(t.RootPath, flags, uint32(t.Permissions))
if err != nil { if err != nil {
return err return err
} }
@ -32,8 +58,10 @@ func (t *FSTree) writeFile(p string, data []byte) error {
if err == nil { if err == nil {
if n == len(data) { if n == len(data) {
err = unix.Linkat(unix.AT_FDCWD, tmpPath, unix.AT_FDCWD, p, unix.AT_SYMLINK_FOLLOW) err = unix.Linkat(unix.AT_FDCWD, tmpPath, unix.AT_FDCWD, p, unix.AT_SYMLINK_FOLLOW)
if err == nil {
w.counter.Inc()
}
if errors.Is(err, unix.EEXIST) { if errors.Is(err, unix.EEXIST) {
// https://github.com/nspcc-dev/neofs-node/issues/2563
err = nil err = nil
} }
} else { } else {

View file

@ -0,0 +1,9 @@
//go:build !linux
package fstree
import "io/fs"
func newSpecificWriteData(_ FileCounter, _ string, _ fs.FileMode, _ bool) func(string, []byte) error {
return nil
}

View file

@ -2,6 +2,7 @@ package shard
import ( import (
"context" "context"
"fmt"
"io/fs" "io/fs"
"math" "math"
"os" "os"
@ -11,7 +12,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
@ -145,8 +145,12 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
require.NoError(t, sh.Close()) require.NoError(t, sh.Close())
addr := object.AddressOf(obj) addr := object.AddressOf(obj)
_, err = fsTree.Put(context.Background(), common.PutPrm{Address: addr, RawData: []byte("not an object")}) // This is copied from `fstree.treePath()` to avoid exporting function just for tests.
require.NoError(t, err) {
saddr := addr.Object().EncodeToString() + "." + addr.Container().EncodeToString()
p := fmt.Sprintf("%s/%s/%s", fsTree.RootPath, saddr[:2], saddr[2:])
require.NoError(t, os.WriteFile(p, []byte("not an object"), fsTree.Permissions))
}
sh = New( sh = New(
WithID(NewIDFromBytes([]byte{})), WithID(NewIDFromBytes([]byte{})),