package fstree import ( "context" "crypto/sha256" "errors" "fmt" "io/fs" "os" "path/filepath" "strconv" "strings" "sync/atomic" "syscall" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) type keyLock interface { Lock(string) Unlock(string) } type noopKeyLock struct{} func (l *noopKeyLock) Lock(string) {} func (l *noopKeyLock) Unlock(string) {} // FSTree represents an object storage as a filesystem tree. type FSTree struct { Info log *logger.Logger *compression.Config Depth uint64 DirNameLen int noSync bool readOnly bool metrics Metrics fileGuard keyLock fileCounter FileCounter fileCounterEnabled bool suffix atomic.Uint64 } // Info groups the information about file storage. type Info struct { // Permission bits of the root directory. Permissions fs.FileMode // Full path to the root directory. RootPath string } const ( // DirNameLen is how many bytes is used to group keys into directories. DirNameLen = 1 // in bytes // MaxDepth is maximum depth of nested directories. MaxDepth = (sha256.Size - 1) / DirNameLen ) var _ common.Storage = (*FSTree)(nil) func New(opts ...Option) *FSTree { f := &FSTree{ Info: Info{ Permissions: 0o700, RootPath: "./", }, Config: nil, Depth: 4, DirNameLen: DirNameLen, metrics: &noopMetrics{}, fileGuard: &noopKeyLock{}, fileCounter: &noopCounter{}, log: &logger.Logger{Logger: zap.L()}, } for i := range opts { opts[i](f) } return f } func stringifyAddress(addr oid.Address) string { return addr.Object().EncodeToString() + "." + addr.Container().EncodeToString() } func addressFromString(s string) (oid.Address, error) { before, after, found := strings.Cut(s, ".") if !found { return oid.Address{}, errors.New("invalid address") } var obj oid.ID if err := obj.DecodeString(before); err != nil { return oid.Address{}, err } var cnr cid.ID if err := cnr.DecodeString(after); err != nil { return oid.Address{}, err } var addr oid.Address addr.SetObject(obj) addr.SetContainer(cnr) return addr, nil } // Iterate iterates over all stored objects. func (t *FSTree) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { var ( err error startedAt = time.Now() ) defer func() { t.metrics.Iterate(time.Since(startedAt), err == nil) }() _, span := tracing.StartSpanFromContext(ctx, "FSTree.Iterate", trace.WithAttributes( attribute.String("path", t.RootPath), attribute.Bool("ignore_errors", prm.IgnoreErrors), )) defer span.End() err = t.iterate(ctx, 0, []string{t.RootPath}, prm) return common.IterateRes{}, err } func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, prm common.IteratePrm) error { curName := strings.Join(curPath[1:], "") dirPath := filepath.Join(curPath...) des, err := os.ReadDir(dirPath) if err != nil { if prm.IgnoreErrors { t.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, zap.String("err", err.Error()), zap.String("directory_path", dirPath)) return nil } return err } isLast := depth >= t.Depth l := len(curPath) curPath = append(curPath, "") for i := range des { select { case <-ctx.Done(): return ctx.Err() default: } curPath[l] = des[i].Name() if !isLast && des[i].IsDir() { err := t.iterate(ctx, depth+1, curPath, prm) if err != nil { // Must be error from handler in case errors are ignored. // Need to report. return err } } if depth != t.Depth { continue } addr, err := addressFromString(curName + des[i].Name()) if err != nil { continue } path := filepath.Join(curPath...) data, err := os.ReadFile(path) if err != nil && os.IsNotExist(err) { continue } if err == nil { data, err = t.Decompress(data) } if err != nil { if prm.IgnoreErrors { t.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, zap.Stringer("address", addr), zap.String("err", err.Error()), zap.String("path", path)) continue } return err } err = prm.Handler(common.IterationElement{ Address: addr, ObjectData: data, StorageID: []byte{}, }) if err != nil { return err } } return nil } func (t *FSTree) treePath(addr oid.Address) string { sAddr := stringifyAddress(addr) var sb strings.Builder sb.Grow(len(t.RootPath) + len(sAddr) + int(t.Depth) + 1) sb.WriteString(t.RootPath) for i := 0; uint64(i) < t.Depth; i++ { sb.WriteRune(filepath.Separator) sb.WriteString(sAddr[:t.DirNameLen]) sAddr = sAddr[t.DirNameLen:] } sb.WriteRune(filepath.Separator) sb.WriteString(sAddr) return sb.String() } // Delete removes the object with the specified address from the storage. func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { var ( err error startedAt = time.Now() ) defer func() { t.metrics.Delete(time.Since(startedAt), err == nil) }() _, span := tracing.StartSpanFromContext(ctx, "FSTree.Delete", trace.WithAttributes( attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), )) defer span.End() if t.readOnly { err = common.ErrReadOnly return common.DeleteRes{}, err } p := t.treePath(prm.Address) if t.fileCounterEnabled { t.fileGuard.Lock(p) err = os.Remove(p) t.fileGuard.Unlock(p) if err == nil { t.fileCounter.Dec() } } else { err = os.Remove(p) } if err != nil && os.IsNotExist(err) { err = logicerr.Wrap(new(apistatus.ObjectNotFound)) } return common.DeleteRes{}, err } // Exists returns the path to the file with object contents if it exists in the storage // and an error otherwise. func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { var ( success = false startedAt = time.Now() ) defer func() { t.metrics.Exists(time.Since(startedAt), success) }() _, span := tracing.StartSpanFromContext(ctx, "FSTree.Exists", trace.WithAttributes( attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), )) defer span.End() select { case <-ctx.Done(): return common.ExistsRes{}, ctx.Err() default: } p := t.treePath(prm.Address) _, err := os.Stat(p) found := err == nil if os.IsNotExist(err) { err = nil } success = err == nil return common.ExistsRes{Exists: found}, err } // Put puts an object in the storage. func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { var ( size int startedAt = time.Now() err error ) defer func() { t.metrics.Put(time.Since(startedAt), size, err == nil) }() _, span := tracing.StartSpanFromContext(ctx, "FSTree.Put", trace.WithAttributes( attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.Bool("dont_compress", prm.DontCompress), )) defer span.End() if t.readOnly { err = common.ErrReadOnly return common.PutRes{}, err } p := t.treePath(prm.Address) if err = util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { if errors.Is(err, syscall.ENOSPC) { err = common.ErrNoSpace return common.PutRes{}, err } return common.PutRes{}, err } if !prm.DontCompress { prm.RawData = t.Compress(prm.RawData) } size = len(prm.RawData) return common.PutRes{StorageID: []byte{}}, t.writeData(p, prm.RawData) } func (t *FSTree) writeData(p string, data []byte) error { tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) return t.writeAndRename(tmpPath, p, data) } // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { if t.fileCounterEnabled { t.fileGuard.Lock(p) defer t.fileGuard.Unlock(p) } err := t.writeFile(tmpPath, data) if err != nil { var pe *fs.PathError if errors.As(err, &pe) { switch pe.Err { case syscall.ENOSPC: err = common.ErrNoSpace _ = os.RemoveAll(tmpPath) } } return err } if t.fileCounterEnabled { t.fileCounter.Inc() var targetFileExists bool if _, e := os.Stat(p); e == nil { targetFileExists = true } err = os.Rename(tmpPath, p) if err == nil && targetFileExists { t.fileCounter.Dec() } } else { err = os.Rename(tmpPath, p) } return err } func (t *FSTree) writeFlags() int { flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL if t.noSync { return flags } return flags | os.O_SYNC } // writeFile writes data to a file with path p. // The code is copied from `os.WriteFile` with minor corrections for flags. func (t *FSTree) writeFile(p string, data []byte) error { f, err := os.OpenFile(p, t.writeFlags(), t.Permissions) if err != nil { return err } _, err = f.Write(data) if err1 := f.Close(); err1 != nil && err == nil { err = err1 } return err } // Get returns an object from the storage by address. func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { var ( startedAt = time.Now() success = false size = 0 ) defer func() { t.metrics.Get(time.Since(startedAt), size, success) }() ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.Get", trace.WithAttributes( attribute.String("path", t.RootPath), attribute.Bool("raw", prm.Raw), attribute.String("address", prm.Address.EncodeToString()), )) defer span.End() p := t.treePath(prm.Address) if _, err := os.Stat(p); os.IsNotExist(err) { return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } var data []byte var err error { _, span := tracing.StartSpanFromContext(ctx, "FSTree.Get.ReadFile") defer span.End() data, err = os.ReadFile(p) if err != nil { if os.IsNotExist(err) { return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } return common.GetRes{}, err } } data, err = t.Decompress(data) if err != nil { return common.GetRes{}, err } size = len(data) obj := objectSDK.New() if err := obj.Unmarshal(data); err != nil { return common.GetRes{}, err } success = true return common.GetRes{Object: obj, RawData: data}, nil } // GetRange implements common.Storage. func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { var ( startedAt = time.Now() success = false size = 0 ) defer func() { t.metrics.GetRange(time.Since(startedAt), size, success) }() ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.GetRange", trace.WithAttributes( attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), attribute.String("length", strconv.FormatUint(prm.Range.GetLength(), 10)), )) defer span.End() res, err := t.Get(ctx, common.GetPrm{Address: prm.Address}) if err != nil { return common.GetRangeRes{}, err } payload := res.Object.Payload() from := prm.Range.GetOffset() to := from + prm.Range.GetLength() if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { return common.GetRangeRes{}, logicerr.Wrap(new(apistatus.ObjectOutOfRange)) } success = true data := payload[from:to] size = len(data) return common.GetRangeRes{ Data: data, }, nil } // initFileCounter walks the file tree rooted at FSTree's root, // counts total items count, inits counter and returns number of stored objects. func (t *FSTree) initFileCounter() error { if !t.fileCounterEnabled { return nil } counter, err := t.countFiles() if err != nil { return err } t.fileCounter.Set(counter) return nil } func (t *FSTree) countFiles() (uint64, error) { var counter uint64 // it is simpler to just consider every file // that is not directory as an object err := filepath.WalkDir(t.RootPath, func(_ string, d fs.DirEntry, _ error) error { if !d.IsDir() { counter++ } return nil }, ) if err != nil { return 0, fmt.Errorf("could not walk through %s directory: %w", t.RootPath, err) } return counter, nil } // Type is fstree storage type used in logs and configuration. const Type = "fstree" // Type implements common.Storage. func (*FSTree) Type() string { return Type } // Path implements common.Storage. func (t *FSTree) Path() string { return t.RootPath } // SetCompressor implements common.Storage. func (t *FSTree) SetCompressor(cc *compression.Config) { t.Config = cc } func (t *FSTree) Compressor() *compression.Config { return t.Config } // SetReportErrorFunc implements common.Storage. func (t *FSTree) SetReportErrorFunc(_ func(string, error)) { // Do nothing, FSTree can encounter only one error which is returned. } func (t *FSTree) SetParentID(parentID string) { t.metrics.SetParentID(parentID) } func (t *FSTree) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { return common.RebuildRes{}, nil }