From 8be5e98e3202c6d5069baf589fff64c4ce1be9b7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 1 Feb 2024 10:26:29 +0300 Subject: [PATCH 1/7] [#970] fstree: Move temporary path handling in a separate function Allow to easier test different implementations. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobstor/fstree/fstree.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index af241becf..3da5568e2 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -357,9 +357,12 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err } size = len(prm.RawData) + return common.PutRes{StorageID: []byte{}}, t.writeData(p, prm.RawData) +} + +func (t *FSTree) writeData(p string, data []byte) error { tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) - err = t.writeAndRename(tmpPath, p, prm.RawData) - return common.PutRes{StorageID: []byte{}}, err + return t.writeAndRename(tmpPath, p, data) } // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. -- 2.45.2 From d77f54e4977aca1e3fd9bffce819c7ec766989c8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 1 Feb 2024 10:28:31 +0300 Subject: [PATCH 2/7] [#970] fstree: Move write functions to a separate file Signed-off-by: Evgenii Stratonikov --- .../blobstor/fstree/fstree.go | 63 ---------------- .../blobstor/fstree/fstree_write_generic.go | 74 +++++++++++++++++++ 2 files changed, 74 insertions(+), 63 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 3da5568e2..7bb8db5da 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -360,69 +360,6 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err return common.PutRes{StorageID: []byte{}}, t.writeData(p, prm.RawData) } -func (t *FSTree) writeData(p string, data []byte) error { - tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) - return t.writeAndRename(tmpPath, p, data) -} - -// writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. -func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { - if t.fileCounterEnabled { - t.fileGuard.Lock(p) - defer t.fileGuard.Unlock(p) - } - - err := t.writeFile(tmpPath, data) - if err != nil { - var pe *fs.PathError - if errors.As(err, &pe) { - switch pe.Err { - case syscall.ENOSPC: - err = common.ErrNoSpace - _ = os.RemoveAll(tmpPath) - } - } - return err - } - - if t.fileCounterEnabled { - t.fileCounter.Inc() - var targetFileExists bool - if _, e := os.Stat(p); e == nil { - targetFileExists = true - } - err = os.Rename(tmpPath, p) - if err == nil && targetFileExists { - t.fileCounter.Dec() - } - } else { - err = os.Rename(tmpPath, p) - } - return err -} - -func (t *FSTree) writeFlags() int { - flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL - if t.noSync { - return flags - } - return flags | os.O_SYNC -} - -// writeFile writes data to a file with path p. -// The code is copied from `os.WriteFile` with minor corrections for flags. -func (t *FSTree) writeFile(p string, data []byte) error { - f, err := os.OpenFile(p, t.writeFlags(), t.Permissions) - if err != nil { - return err - } - _, err = f.Write(data) - if err1 := f.Close(); err1 != nil && err == nil { - err = err1 - } - return err -} - // Get returns an object from the storage by address. func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { var ( diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go new file mode 100644 index 000000000..411914034 --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -0,0 +1,74 @@ +package fstree + +import ( + "errors" + "io/fs" + "os" + "strconv" + "syscall" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" +) + +func (t *FSTree) writeData(p string, data []byte) error { + tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) + return t.writeAndRename(tmpPath, p, data) +} + +// writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. +func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { + if t.fileCounterEnabled { + t.fileGuard.Lock(p) + defer t.fileGuard.Unlock(p) + } + + err := t.writeFile(tmpPath, data) + if err != nil { + var pe *fs.PathError + if errors.As(err, &pe) { + switch pe.Err { + case syscall.ENOSPC: + err = common.ErrNoSpace + _ = os.RemoveAll(tmpPath) + } + } + return err + } + + if t.fileCounterEnabled { + t.fileCounter.Inc() + var targetFileExists bool + if _, e := os.Stat(p); e == nil { + targetFileExists = true + } + err = os.Rename(tmpPath, p) + if err == nil && targetFileExists { + t.fileCounter.Dec() + } + } else { + err = os.Rename(tmpPath, p) + } + return err +} + +func (t *FSTree) writeFlags() int { + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL + if t.noSync { + return flags + } + return flags | os.O_SYNC +} + +// writeFile writes data to a file with path p. +// The code is copied from `os.WriteFile` with minor corrections for flags. +func (t *FSTree) writeFile(p string, data []byte) error { + f, err := os.OpenFile(p, t.writeFlags(), t.Permissions) + if err != nil { + return err + } + _, err = f.Write(data) + if err1 := f.Close(); err1 != nil && err == nil { + err = err1 + } + return err +} -- 2.45.2 From 8a49ace9b61790da9788ed2a636c9233e2da9b89 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Sat, 9 Sep 2023 11:16:06 +0300 Subject: [PATCH 3/7] [#970] fstree: Add linux-specific file writer using O_TMPFILE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit O_TMPFILE is implemented for all modern FSes and it's much easier and safer to use. If application crashes in the middle of writing this file would be gone and won't leave any garbage. Notice that this implementation makes a different choice wrt EEXIST handling, generic one always overwrites, while this one keeps the old data. There is no real performance difference. SSD (slow&old), XFS, Core i7-8565U: Sync ``` name old time/op new time/op delta Put/size=1024,thread=1/fstree-8 1.74ms ± 3% 0.06ms ± 7% -96.31% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 10.0ms ±41% 1.1ms ±18% -88.95% (p=0.000 n=9+10) Put/size=1024,thread=100/fstree-8 32.3ms ±60% 6.5ms ±14% -79.97% (p=0.000 n=10+10) Put/size=1048576,thread=1/fstree-8 17.8ms ±90% 3.4ms ±70% -81.08% (p=0.000 n=10+10) Put/size=1048576,thread=20/fstree-8 103ms ±174% 112ms ±158% ~ (p=0.971 n=10+10) Put/size=1048576,thread=100/fstree-8 949ms ±78% 583ms ±132% ~ (p=0.089 n=10+10) name old alloc/op new alloc/op delta Put/size=1024,thread=1/fstree-8 3.17kB ± 1% 1.96kB ± 0% -38.09% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 59.6kB ± 1% 39.2kB ± 1% -34.30% (p=0.000 n=8+10) Put/size=1024,thread=100/fstree-8 299kB ± 0% 198kB ± 0% -33.90% (p=0.000 n=7+9) Put/size=1048576,thread=1/fstree-8 3.38kB ± 1% 2.36kB ± 1% -30.22% (p=0.000 n=10+10) Put/size=1048576,thread=20/fstree-8 65.7kB ± 4% 47.7kB ± 6% -27.27% (p=0.000 n=10+10) Put/size=1048576,thread=100/fstree-8 351kB ± 8% 245kB ± 8% -30.22% (p=0.000 n=10+10) name old allocs/op new allocs/op delta Put/size=1024,thread=1/fstree-8 30.3 ± 2% 21.0 ± 0% -30.69% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 554 ± 1% 413 ± 0% -25.35% (p=0.000 n=8+10) Put/size=1024,thread=100/fstree-8 2.77k ± 0% 2.07k ± 0% -25.27% (p=0.000 n=7+10) Put/size=1048576,thread=1/fstree-8 32.0 ± 0% 25.0 ± 0% -21.88% (p=0.000 n=9+8) Put/size=1048576,thread=20/fstree-8 609 ± 5% 494 ± 6% -18.93% (p=0.000 n=10+10) Put/size=1048576,thread=100/fstree-8 3.25k ± 9% 2.50k ± 8% -23.21% (p=0.000 n=10+10) ``` No sync ``` name old time/op new time/op delta Put/size=1024,thread=1/fstree-8 71.3µs ±10% 59.8µs ±10% -16.21% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 1.43ms ± 6% 1.22ms ±13% -14.53% (p=0.000 n=10+10) Put/size=1024,thread=100/fstree-8 8.12ms ± 3% 6.36ms ± 2% -21.67% (p=0.000 n=8+9) Put/size=1048576,thread=1/fstree-8 1.88ms ±70% 1.61ms ±78% ~ (p=0.393 n=10+10) Put/size=1048576,thread=20/fstree-8 32.7ms ±28% 34.2ms ±112% ~ (p=0.968 n=9+10) Put/size=1048576,thread=100/fstree-8 262ms ±56% 226ms ±34% ~ (p=0.447 n=10+9) name old alloc/op new alloc/op delta Put/size=1024,thread=1/fstree-8 2.89kB ± 0% 1.96kB ± 0% -32.28% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 58.2kB ± 0% 39.5kB ± 0% -32.09% (p=0.000 n=8+8) Put/size=1024,thread=100/fstree-8 291kB ± 0% 198kB ± 0% -32.19% (p=0.000 n=9+9) Put/size=1048576,thread=1/fstree-8 3.05kB ± 1% 2.13kB ± 1% -30.16% (p=0.000 n=10+9) Put/size=1048576,thread=20/fstree-8 62.6kB ± 0% 44.3kB ± 0% -29.23% (p=0.000 n=9+9) Put/size=1048576,thread=100/fstree-8 302kB ± 0% 210kB ± 1% -30.39% (p=0.000 n=9+9) name old allocs/op new allocs/op delta Put/size=1024,thread=1/fstree-8 27.0 ± 0% 21.0 ± 0% -22.22% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 539 ± 0% 415 ± 0% -22.98% (p=0.000 n=10+10) Put/size=1024,thread=100/fstree-8 2.69k ± 0% 2.07k ± 0% -23.09% (p=0.000 n=9+9) Put/size=1048576,thread=1/fstree-8 28.0 ± 0% 22.3 ± 3% -20.36% (p=0.000 n=8+10) Put/size=1048576,thread=20/fstree-8 577 ± 0% 458 ± 0% -20.72% (p=0.000 n=9+9) Put/size=1048576,thread=100/fstree-8 2.76k ± 0% 2.15k ± 0% -22.05% (p=0.000 n=9+8) ``` HDD (LVM), ext4, Ryzen 5 1600: Sync ``` │ fs.sync-generic │ fs.sync-linux │ │ sec/op │ sec/op vs base │ Put/size=1024,thread=1/fstree-12 34.70m ± 19% 33.59m ± 16% ~ (p=0.529 n=10) Put/size=1024,thread=20/fstree-12 188.8m ± 8% 189.2m ± 16% ~ (p=0.739 n=10) Put/size=1024,thread=100/fstree-12 264.8m ± 22% 273.6m ± 28% ~ (p=0.353 n=10) Put/size=1048576,thread=1/fstree-12 54.90m ± 14% 47.08m ± 18% ~ (p=0.063 n=10) Put/size=1048576,thread=20/fstree-12 244.1m ± 14% 220.4m ± 22% ~ (p=0.579 n=10) Put/size=1048576,thread=100/fstree-12 847.2m ± 5% 893.6m ± 3% +5.48% (p=0.000 n=10) geomean 164.3m 158.9m -3.29% │ fs.sync-generic │ fs.sync-linux │ │ B/op │ B/op vs base │ Put/size=1024,thread=1/fstree-12 3.375Ki ± 1% 2.471Ki ± 1% -26.80% (p=0.000 n=10) Put/size=1024,thread=20/fstree-12 66.62Ki ± 6% 49.21Ki ± 6% -26.15% (p=0.000 n=10) Put/size=1024,thread=100/fstree-12 319.2Ki ± 1% 230.9Ki ± 2% -27.64% (p=0.000 n=10) Put/size=1048576,thread=1/fstree-12 3.457Ki ± 1% 2.559Ki ± 1% -25.97% (p=0.000 n=10) Put/size=1048576,thread=20/fstree-12 66.91Ki ± 1% 49.16Ki ± 1% -26.52% (p=0.000 n=10) Put/size=1048576,thread=100/fstree-12 338.8Ki ± 2% 252.3Ki ± 3% -25.54% (p=0.000 n=10) geomean 42.17Ki 31.02Ki -26.44% │ fs.sync-generic │ fs.sync-linux │ │ allocs/op │ allocs/op vs base │ Put/size=1024,thread=1/fstree-12 33.00 ± 0% 27.00 ± 0% -18.18% (p=0.000 n=10) Put/size=1024,thread=20/fstree-12 639.5 ± 1% 519.0 ± 2% -18.84% (p=0.000 n=10) Put/size=1024,thread=100/fstree-12 3.059k ± 1% 2.478k ± 2% -18.99% (p=0.000 n=10) Put/size=1048576,thread=1/fstree-12 33.50 ± 1% 28.00 ± 4% -16.42% (p=0.000 n=10) Put/size=1048576,thread=20/fstree-12 638.5 ± 1% 520.0 ± 1% -18.56% (p=0.000 n=10) Put/size=1048576,thread=100/fstree-12 3.209k ± 2% 2.655k ± 2% -17.28% (p=0.000 n=10) geomean 405.3 332.1 -18.05% ``` No sync ``` │ fs.nosync-generic │ fs.nosync-linux │ │ sec/op │ sec/op vs base │ Put/size=1024,thread=1/fstree-12 148.2µ ± 20% 136.6µ ± 19% -7.89% (p=0.029 n=10) Put/size=1024,thread=20/fstree-12 1.140m ± 26% 1.364m ± 16% ~ (p=0.143 n=10) Put/size=1024,thread=100/fstree-12 11.93m ± 68% 26.89m ± 62% ~ (p=0.123 n=10) Put/size=1048576,thread=1/fstree-12 1.302m ± 3% 1.287m ± 5% ~ (p=0.481 n=10) Put/size=1048576,thread=20/fstree-12 77.52m ± 8% 74.07m ± 7% ~ (p=0.278 n=10+9) Put/size=1048576,thread=100/fstree-12 226.1m ± ∞ ¹ geomean 5.986m 3.434m +18.60% ² ¹ need >= 6 samples for confidence interval at level 0.95 ² benchmark set differs from baseline; geomeans may not be comparable │ fs.nosync-generic │ fs.nosync-linux │ │ B/op │ B/op vs base │ Put/size=1024,thread=1/fstree-12 2.879Ki ± 0% 1.972Ki ± 0% -31.51% (p=0.000 n=10) Put/size=1024,thread=20/fstree-12 55.94Ki ± 1% 37.90Ki ± 1% -32.25% (p=0.000 n=10) Put/size=1024,thread=100/fstree-12 272.6Ki ± 0% 182.1Ki ± 9% -33.21% (p=0.000 n=10) Put/size=1048576,thread=1/fstree-12 3.158Ki ± 0% 2.259Ki ± 0% -28.46% (p=0.000 n=10) Put/size=1048576,thread=20/fstree-12 58.87Ki ± 0% 41.03Ki ± 0% -30.30% (p=0.000 n=10+9) Put/size=1048576,thread=100/fstree-12 299.8Ki ± ∞ ¹ geomean 36.71Ki 16.60Ki -31.17% ² ¹ need >= 6 samples for confidence interval at level 0.95 ² benchmark set differs from baseline; geomeans may not be comparable │ fs.nosync-generic │ fs.nosync-linux │ │ allocs/op │ allocs/op vs base │ Put/size=1024,thread=1/fstree-12 28.00 ± 0% 22.00 ± 0% -21.43% (p=0.000 n=10) Put/size=1024,thread=20/fstree-12 530.0 ± 0% 407.5 ± 1% -23.11% (p=0.000 n=10) Put/size=1024,thread=100/fstree-12 2.567k ± 0% 1.956k ± 9% -23.77% (p=0.000 n=10) Put/size=1048576,thread=1/fstree-12 30.00 ± 0% 24.00 ± 0% -20.00% (p=0.000 n=10) Put/size=1048576,thread=20/fstree-12 553.5 ± 0% 434.0 ± 0% -21.59% (n=10+9) Put/size=1048576,thread=100/fstree-12 2.803k ± ∞ ¹ geomean 347.9 178.8 -21.99% ² ¹ need >= 6 samples for confidence interval at level 0.95 ² benchmark set differs from baseline; geomeans may not be comparable ``` Signed-off-by: Roman Khimov Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- .../blobstor/fstree/fstree_write_generic.go | 2 + .../blobstor/fstree/fstree_write_linux.go | 48 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go diff --git a/go.mod b/go.mod index 5d6edc222..06d2d3299 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,7 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/sync v0.6.0 + golang.org/x/sys v0.16.0 golang.org/x/term v0.16.0 google.golang.org/grpc v1.61.0 google.golang.org/protobuf v1.32.0 @@ -120,7 +121,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.18.0 // indirect golang.org/x/net v0.20.0 // indirect - golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 411914034..0882be0dd 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -1,3 +1,5 @@ +//go:build !linux + package fstree import ( diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go new file mode 100644 index 000000000..edb7f0dce --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -0,0 +1,48 @@ +//go:build linux + +package fstree + +import ( + "errors" + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "golang.org/x/sys/unix" +) + +func (t *FSTree) writeData(p string, data []byte) error { + err := t.writeFile(p, data) + if errors.Is(err, unix.ENOSPC) { + return common.ErrNoSpace + } + return err +} + +func (t *FSTree) writeFile(p string, data []byte) error { + flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC + if !t.noSync { + flags |= unix.O_DSYNC + } + fd, err := unix.Open(t.RootPath, flags, uint32(t.Permissions)) + if err != nil { + return err + } + tmpPath := "/proc/self/fd/" + strconv.FormatUint(uint64(fd), 10) + n, err := unix.Write(fd, data) + if err == nil { + if n == len(data) { + err = unix.Linkat(unix.AT_FDCWD, tmpPath, unix.AT_FDCWD, p, unix.AT_SYMLINK_FOLLOW) + if errors.Is(err, unix.EEXIST) { + // https://github.com/nspcc-dev/neofs-node/issues/2563 + err = nil + } + } else { + err = errors.New("incomplete write") + } + } + errClose := unix.Close(fd) + if err != nil { + return err // Close() error is ignored, we have a better one. + } + return errClose +} -- 2.45.2 From 8b7d50708bc393cdd78d75b3533fabc34c8b5e92 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 11:57:23 +0300 Subject: [PATCH 4/7] [#970] fstree: Handle unsupported O_TMPFILE Metabase test relied on this behaviour, so fix the test too. Cherry-picking was hard and did too many conflicts, here is an original PR: https://github.com/nspcc-dev/neofs-node/pull/2624 Signed-off-by: Evgenii Stratonikov --- .../blobstor/fstree/control.go | 7 +++ .../blobstor/fstree/fstree.go | 4 +- .../blobstor/fstree/fstree_write_generic.go | 58 +++++++++++-------- .../blobstor/fstree/fstree_write_linux.go | 46 ++++++++++++--- .../blobstor/fstree/fstree_write_specific.go | 9 +++ .../shard/control_test.go | 10 +++- 6 files changed, 97 insertions(+), 37 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index c56312d38..b126f8d29 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -16,6 +16,13 @@ func (t *FSTree) Init() error { if err := util.MkdirAllX(t.RootPath, t.Permissions); err != nil { return err } + if !t.readOnly { + f := newSpecificWriteData(t.fileCounter, t.RootPath, t.Permissions, t.noSync) + if f != nil { + t.writeData = f + } + } + return t.initFileCounter() } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 7bb8db5da..fc29d4eaa 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -10,7 +10,6 @@ import ( "path/filepath" "strconv" "strings" - "sync/atomic" "syscall" "time" @@ -58,7 +57,7 @@ type FSTree struct { fileCounter FileCounter fileCounterEnabled bool - suffix atomic.Uint64 + writeData func(string, []byte) error } // Info groups the information about file storage. @@ -96,6 +95,7 @@ func New(opts ...Option) *FSTree { for i := range opts { opts[i](f) } + f.writeData = newGenericWriteData(f) return f } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 0882be0dd..3d65808ec 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -1,5 +1,3 @@ -//go:build !linux - package fstree import ( @@ -7,24 +5,46 @@ import ( "io/fs" "os" "strconv" + "sync/atomic" "syscall" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" ) -func (t *FSTree) writeData(p string, data []byte) error { - tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) - return t.writeAndRename(tmpPath, p, data) +type genericWriter struct { + perm fs.FileMode + flags int + + t *FSTree + suffix atomic.Uint64 +} + +func newGenericWriteData(t *FSTree) func(string, []byte) error { + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL + if !t.noSync { + flags |= os.O_SYNC + } + var w = &genericWriter{ + perm: t.Permissions, + flags: flags, + t: t, + } + return w.writeData +} + +func (w *genericWriter) writeData(p string, data []byte) error { + tmpPath := p + "#" + strconv.FormatUint(w.suffix.Add(1), 10) + return w.writeAndRename(tmpPath, p, data) } // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. -func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { - if t.fileCounterEnabled { - t.fileGuard.Lock(p) - defer t.fileGuard.Unlock(p) +func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error { + if w.t.fileCounterEnabled { + w.t.fileGuard.Lock(p) + defer w.t.fileGuard.Unlock(p) } - err := t.writeFile(tmpPath, data) + err := w.writeFile(tmpPath, data) if err != nil { var pe *fs.PathError if errors.As(err, &pe) { @@ -37,15 +57,15 @@ func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { return err } - if t.fileCounterEnabled { - t.fileCounter.Inc() + if w.t.fileCounterEnabled { + w.t.fileCounter.Inc() var targetFileExists bool if _, e := os.Stat(p); e == nil { targetFileExists = true } err = os.Rename(tmpPath, p) if err == nil && targetFileExists { - t.fileCounter.Dec() + w.t.fileCounter.Dec() } } else { err = os.Rename(tmpPath, p) @@ -53,18 +73,10 @@ func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { return err } -func (t *FSTree) writeFlags() int { - flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL - if t.noSync { - return flags - } - return flags | os.O_SYNC -} - // writeFile writes data to a file with path p. // The code is copied from `os.WriteFile` with minor corrections for flags. -func (t *FSTree) writeFile(p string, data []byte) error { - f, err := os.OpenFile(p, t.writeFlags(), t.Permissions) +func (w *genericWriter) writeFile(p string, data []byte) error { + f, err := os.OpenFile(p, w.flags, w.perm) if err != nil { return err } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go index edb7f0dce..3416cf7ac 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -4,26 +4,52 @@ package fstree import ( "errors" + "io/fs" "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "golang.org/x/sys/unix" ) -func (t *FSTree) writeData(p string, data []byte) error { - err := t.writeFile(p, data) +type linuxWriter struct { + root string + perm uint32 + flags int + + counter FileCounter +} + +func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync bool) func(string, []byte) error { + flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC + if !noSync { + flags |= unix.O_DSYNC + } + fd, err := unix.Open(root, flags, uint32(perm)) + if err != nil { + // Which means that OS-specific writeData can't be created + // and FSTree should use the generic one. + return nil + } + _ = unix.Close(fd) // Don't care about error. + w := &linuxWriter{ + root: root, + perm: uint32(perm), + flags: flags, + counter: c, + } + return w.writeData +} + +func (w *linuxWriter) writeData(p string, data []byte) error { + err := w.writeFile(p, data) if errors.Is(err, unix.ENOSPC) { return common.ErrNoSpace } return err } -func (t *FSTree) writeFile(p string, data []byte) error { - flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC - if !t.noSync { - flags |= unix.O_DSYNC - } - fd, err := unix.Open(t.RootPath, flags, uint32(t.Permissions)) +func (w *linuxWriter) writeFile(p string, data []byte) error { + fd, err := unix.Open(w.root, w.flags, w.perm) if err != nil { return err } @@ -32,8 +58,10 @@ func (t *FSTree) writeFile(p string, data []byte) error { if err == nil { if n == len(data) { err = unix.Linkat(unix.AT_FDCWD, tmpPath, unix.AT_FDCWD, p, unix.AT_SYMLINK_FOLLOW) + if err == nil { + w.counter.Inc() + } if errors.Is(err, unix.EEXIST) { - // https://github.com/nspcc-dev/neofs-node/issues/2563 err = nil } } else { diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go new file mode 100644 index 000000000..16b630b6a --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go @@ -0,0 +1,9 @@ +//go:build !linux + +package fstree + +import "io/fs" + +func newSpecificWriteData(_ FileCounter, _ string, _ fs.FileMode, _ bool) func(string, []byte) error { + return nil +} diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 4d7354595..d08747e13 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -2,6 +2,7 @@ package shard import ( "context" + "fmt" "io/fs" "math" "os" @@ -11,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -145,8 +145,12 @@ func TestRefillMetabaseCorrupted(t *testing.T) { require.NoError(t, sh.Close()) addr := object.AddressOf(obj) - _, err = fsTree.Put(context.Background(), common.PutPrm{Address: addr, RawData: []byte("not an object")}) - require.NoError(t, err) + // This is copied from `fstree.treePath()` to avoid exporting function just for tests. + { + saddr := addr.Object().EncodeToString() + "." + addr.Container().EncodeToString() + p := fmt.Sprintf("%s/%s/%s", fsTree.RootPath, saddr[:2], saddr[2:]) + require.NoError(t, os.WriteFile(p, []byte("not an object"), fsTree.Permissions)) + } sh = New( WithID(NewIDFromBytes([]byte{})), -- 2.45.2 From 67a45b4e680d7a0ad809d5c4cc524fde6151f285 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 17:53:27 +0300 Subject: [PATCH 5/7] [#970] fstree: Move delete implementation to a separate file Signed-off-by: Evgenii Stratonikov --- .../blobstor/fstree/control.go | 2 +- .../blobstor/fstree/fstree.go | 22 +++----------- .../blobstor/fstree/fstree_write_generic.go | 30 +++++++++++++++++-- .../blobstor/fstree/fstree_write_linux.go | 17 +++++++++-- .../blobstor/fstree/fstree_write_specific.go | 6 ++-- 5 files changed, 52 insertions(+), 25 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index b126f8d29..cec531f2e 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -19,7 +19,7 @@ func (t *FSTree) Init() error { if !t.readOnly { f := newSpecificWriteData(t.fileCounter, t.RootPath, t.Permissions, t.noSync) if f != nil { - t.writeData = f + t.writer = f } } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index fc29d4eaa..ba691b056 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -57,7 +57,7 @@ type FSTree struct { fileCounter FileCounter fileCounterEnabled bool - writeData func(string, []byte) error + writer writer } // Info groups the information about file storage. @@ -95,7 +95,7 @@ func New(opts ...Option) *FSTree { for i := range opts { opts[i](f) } - f.writeData = newGenericWriteData(f) + f.writer = newGenericWriteData(f) return f } @@ -266,21 +266,7 @@ func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Delet } p := t.treePath(prm.Address) - - if t.fileCounterEnabled { - t.fileGuard.Lock(p) - err = os.Remove(p) - t.fileGuard.Unlock(p) - if err == nil { - t.fileCounter.Dec() - } - } else { - err = os.Remove(p) - } - - if err != nil && os.IsNotExist(err) { - err = logicerr.Wrap(new(apistatus.ObjectNotFound)) - } + err = t.writer.removeFile(p) return common.DeleteRes{}, err } @@ -357,7 +343,7 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err } size = len(prm.RawData) - return common.PutRes{StorageID: []byte{}}, t.writeData(p, prm.RawData) + return common.PutRes{StorageID: []byte{}}, t.writer.writeData(p, prm.RawData) } // Get returns an object from the storage by address. diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 3d65808ec..e1e3a53bb 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -9,8 +9,15 @@ import ( "syscall" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) +type writer interface { + writeData(string, []byte) error + removeFile(string) error +} + type genericWriter struct { perm fs.FileMode flags int @@ -19,7 +26,7 @@ type genericWriter struct { suffix atomic.Uint64 } -func newGenericWriteData(t *FSTree) func(string, []byte) error { +func newGenericWriteData(t *FSTree) writer { flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL if !t.noSync { flags |= os.O_SYNC @@ -29,7 +36,7 @@ func newGenericWriteData(t *FSTree) func(string, []byte) error { flags: flags, t: t, } - return w.writeData + return w } func (w *genericWriter) writeData(p string, data []byte) error { @@ -86,3 +93,22 @@ func (w *genericWriter) writeFile(p string, data []byte) error { } return err } + +func (w *genericWriter) removeFile(p string) error { + var err error + if w.t.fileCounterEnabled { + w.t.fileGuard.Lock(p) + err = os.Remove(p) + w.t.fileGuard.Unlock(p) + if err == nil { + w.t.fileCounter.Dec() + } + } else { + err = os.Remove(p) + } + + if err != nil && os.IsNotExist(err) { + err = logicerr.Wrap(new(apistatus.ObjectNotFound)) + } + return err +} diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go index 3416cf7ac..a77c9d1b2 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -8,6 +8,8 @@ import ( "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "golang.org/x/sys/unix" ) @@ -19,7 +21,7 @@ type linuxWriter struct { counter FileCounter } -func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync bool) func(string, []byte) error { +func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync bool) writer { flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC if !noSync { flags |= unix.O_DSYNC @@ -37,7 +39,7 @@ func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync b flags: flags, counter: c, } - return w.writeData + return w } func (w *linuxWriter) writeData(p string, data []byte) error { @@ -74,3 +76,14 @@ func (w *linuxWriter) writeFile(p string, data []byte) error { } return errClose } + +func (w *linuxWriter) removeFile(p string) error { + err := unix.Unlink(p) + if err != nil && err == unix.ENOENT { + return logicerr.Wrap(new(apistatus.ObjectNotFound)) + } + if err == nil { + w.counter.Dec() + } + return err +} diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go index 16b630b6a..9bd559366 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go @@ -2,8 +2,10 @@ package fstree -import "io/fs" +import ( + "io/fs" +) -func newSpecificWriteData(_ FileCounter, _ string, _ fs.FileMode, _ bool) func(string, []byte) error { +func newSpecificWriteData(_ FileCounter, _ string, _ fs.FileMode, _ bool) writer { return nil } -- 2.45.2 From 1761f2538bc3eef3d681c9fa0a0cc2ebfdce020a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 18:04:18 +0300 Subject: [PATCH 6/7] [#970] fstree: Move file locking to the generic writer It is not a part of FSTree itself, but rather a way to solve concurrent counter update on non-linux implementations. New linux implementations is pretty simple: link fails when the file exists, unlink fails when the file doesn't exist. Signed-off-by: Evgenii Stratonikov --- .../blobstor/fstree/counter.go | 5 +++ .../blobstor/fstree/fstree.go | 9 ++-- .../blobstor/fstree/fstree_write_generic.go | 45 ++++++++++++------- .../blobstor/fstree/option.go | 3 -- 4 files changed, 37 insertions(+), 25 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/counter.go b/pkg/local_object_storage/blobstor/fstree/counter.go index 70b346093..718104e2e 100644 --- a/pkg/local_object_storage/blobstor/fstree/counter.go +++ b/pkg/local_object_storage/blobstor/fstree/counter.go @@ -18,6 +18,11 @@ func (c *noopCounter) Set(uint64) {} func (c *noopCounter) Inc() {} func (c *noopCounter) Dec() {} +func counterEnabled(c FileCounter) bool { + _, noop := c.(*noopCounter) + return !noop +} + type SimpleCounter struct { v atomic.Uint64 } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index ba691b056..76ce5bf17 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -53,9 +53,7 @@ type FSTree struct { readOnly bool metrics Metrics - fileGuard keyLock - fileCounter FileCounter - fileCounterEnabled bool + fileCounter FileCounter writer writer } @@ -88,14 +86,13 @@ func New(opts ...Option) *FSTree { Depth: 4, DirNameLen: DirNameLen, metrics: &noopMetrics{}, - fileGuard: &noopKeyLock{}, fileCounter: &noopCounter{}, log: &logger.Logger{Logger: zap.L()}, } for i := range opts { opts[i](f) } - f.writer = newGenericWriteData(f) + f.writer = newGenericWriteData(f.fileCounter, f.Permissions, f.noSync) return f } @@ -444,7 +441,7 @@ func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.G // initFileCounter walks the file tree rooted at FSTree's root, // counts total items count, inits counter and returns number of stored objects. func (t *FSTree) initFileCounter() error { - if !t.fileCounterEnabled { + if !counterEnabled(t.fileCounter) { return nil } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index e1e3a53bb..808076959 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) @@ -22,19 +23,31 @@ type genericWriter struct { perm fs.FileMode flags int - t *FSTree - suffix atomic.Uint64 + fileGuard keyLock + fileCounter FileCounter + fileCounterEnabled bool + suffix atomic.Uint64 } -func newGenericWriteData(t *FSTree) writer { +func newGenericWriteData(c FileCounter, perm fs.FileMode, noSync bool) writer { flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL - if !t.noSync { + if !noSync { flags |= os.O_SYNC } + + var fileGuard keyLock = &noopKeyLock{} + fileCounterEnabled := counterEnabled(c) + if fileCounterEnabled { + fileGuard = utilSync.NewKeyLocker[string]() + } + var w = &genericWriter{ - perm: t.Permissions, + perm: perm, flags: flags, - t: t, + + fileCounterEnabled: fileCounterEnabled, + fileGuard: fileGuard, + fileCounter: c, } return w } @@ -46,9 +59,9 @@ func (w *genericWriter) writeData(p string, data []byte) error { // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error { - if w.t.fileCounterEnabled { - w.t.fileGuard.Lock(p) - defer w.t.fileGuard.Unlock(p) + if w.fileCounterEnabled { + w.fileGuard.Lock(p) + defer w.fileGuard.Unlock(p) } err := w.writeFile(tmpPath, data) @@ -64,15 +77,15 @@ func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error { return err } - if w.t.fileCounterEnabled { - w.t.fileCounter.Inc() + if w.fileCounterEnabled { + w.fileCounter.Inc() var targetFileExists bool if _, e := os.Stat(p); e == nil { targetFileExists = true } err = os.Rename(tmpPath, p) if err == nil && targetFileExists { - w.t.fileCounter.Dec() + w.fileCounter.Dec() } } else { err = os.Rename(tmpPath, p) @@ -96,12 +109,12 @@ func (w *genericWriter) writeFile(p string, data []byte) error { func (w *genericWriter) removeFile(p string) error { var err error - if w.t.fileCounterEnabled { - w.t.fileGuard.Lock(p) + if w.fileCounterEnabled { + w.fileGuard.Lock(p) err = os.Remove(p) - w.t.fileGuard.Unlock(p) + w.fileGuard.Unlock(p) if err == nil { - w.t.fileCounter.Dec() + w.fileCounter.Dec() } } else { err = os.Remove(p) diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go index e6c7a2608..4d1f8fc22 100644 --- a/pkg/local_object_storage/blobstor/fstree/option.go +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -4,7 +4,6 @@ import ( "io/fs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" "go.uber.org/zap" ) @@ -48,9 +47,7 @@ func WithMetrics(m Metrics) Option { func WithFileCounter(c FileCounter) Option { return func(f *FSTree) { - f.fileCounterEnabled = true f.fileCounter = c - f.fileGuard = utilSync.NewKeyLocker[string]() } } -- 2.45.2 From d14d065fdc02d3db5fe82f0d882a7b7f517d9518 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 21:07:02 +0300 Subject: [PATCH 7/7] [#970] fstree: Add build tag to enable generic version on linux Unless tested, generic version can start gaining bugs. With a separate build tag we can have the best of both worlds: 1. Use optimized implementation for linux by default. 2. Run tests or benchmarks for both. Note that they are not actually run automatically now, but this is at leas possible. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go | 2 +- .../blobstor/fstree/fstree_write_specific.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go index a77c9d1b2..efc5a3d3d 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build linux && !fstree_generic package fstree diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go index 9bd559366..67052d947 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go @@ -1,4 +1,4 @@ -//go:build !linux +//go:build !linux || fstree_generic package fstree -- 2.45.2