package fstree import ( "crypto/sha256" "errors" "fmt" "io/fs" "os" "path/filepath" "strings" "github.com/nspcc-dev/neofs-node/pkg/util" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address" oidSDK "github.com/nspcc-dev/neofs-sdk-go/object/id" ) // FSTree represents an object storage as a filesystem tree. type FSTree struct { Info Depth int DirNameLen int } // Info groups the information about file storage. type Info struct { // Permission bits of the root directory. Permissions fs.FileMode // Full path to the root directory. RootPath string } const ( // DirNameLen is how many bytes is used to group keys into directories. DirNameLen = 1 // in bytes // MaxDepth is maximum depth of nested directories. MaxDepth = (sha256.Size - 1) / DirNameLen ) // ErrFileNotFound is returned when file is missing. var ErrFileNotFound = errors.New("file not found") func stringifyAddress(addr *addressSDK.Address) string { id, _ := addr.ObjectID() cnr, _ := addr.ContainerID() return id.EncodeToString() + "." + cnr.EncodeToString() } func addressFromString(s string) (*addressSDK.Address, error) { ss := strings.SplitN(s, ".", 2) if len(ss) != 2 { return nil, errors.New("invalid address") } var oid oidSDK.ID if err := oid.DecodeString(ss[0]); err != nil { return nil, err } var id cid.ID if err := id.DecodeString(ss[1]); err != nil { return nil, err } addr := addressSDK.NewAddress() addr.SetObjectID(oid) addr.SetContainerID(id) return addr, nil } // IterationPrm contains iteraction parameters. type IterationPrm struct { handler func(addr *addressSDK.Address, data []byte) error ignoreErrors bool } // WithHandler sets a function to call on each object. func (p *IterationPrm) WithHandler(f func(addr *addressSDK.Address, data []byte) error) *IterationPrm { p.handler = f return p } // WithIgnoreErrors sets a flag indicating whether errors should be ignored. func (p *IterationPrm) WithIgnoreErrors(ignore bool) *IterationPrm { p.ignoreErrors = ignore return p } // Iterate iterates over all stored objects. func (t *FSTree) Iterate(prm *IterationPrm) error { return t.iterate(0, []string{t.RootPath}, prm) } func (t *FSTree) iterate(depth int, curPath []string, prm *IterationPrm) error { curName := strings.Join(curPath[1:], "") des, err := os.ReadDir(filepath.Join(curPath...)) if err != nil { if prm.ignoreErrors { return nil } return err } isLast := depth >= t.Depth l := len(curPath) curPath = append(curPath, "") for i := range des { curPath[l] = des[i].Name() if !isLast && des[i].IsDir() { err := t.iterate(depth+1, curPath, prm) if err != nil { // Must be error from handler in case errors are ignored. // Need to report. return err } } if depth != t.Depth { continue } addr, err := addressFromString(curName + des[i].Name()) if err != nil { continue } data, err := os.ReadFile(filepath.Join(curPath...)) if err != nil { if prm.ignoreErrors { continue } return err } if err := prm.handler(addr, data); err != nil { // Error occurred in handler, outside of our scope, needs to be reported. return err } } return nil } func (t *FSTree) treePath(addr *addressSDK.Address) string { sAddr := stringifyAddress(addr) dirs := make([]string, 0, t.Depth+1+1) // 1 for root, 1 for file dirs = append(dirs, t.RootPath) for i := 0; i < t.Depth; i++ { dirs = append(dirs, sAddr[:t.DirNameLen]) sAddr = sAddr[t.DirNameLen:] } dirs = append(dirs, sAddr) return filepath.Join(dirs...) } // Delete removes the object with the specified address from the storage. func (t *FSTree) Delete(addr *addressSDK.Address) error { p, err := t.Exists(addr) if err != nil { return err } return os.Remove(p) } // Exists returns the path to the file with object contents if it exists in the storage // and an error otherwise. func (t *FSTree) Exists(addr *addressSDK.Address) (string, error) { p := t.treePath(addr) _, err := os.Stat(p) if os.IsNotExist(err) { err = ErrFileNotFound } return p, err } // Put puts an object in the storage. func (t *FSTree) Put(addr *addressSDK.Address, data []byte) error { p := t.treePath(addr) if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { return err } return os.WriteFile(p, data, t.Permissions) } // PutStream puts executes handler on a file opened for write. func (t *FSTree) PutStream(addr *addressSDK.Address, handler func(*os.File) error) error { p := t.treePath(addr) if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { return err } f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, t.Permissions) if err != nil { return err } defer f.Close() return handler(f) } // Get returns an object from the storage by address. func (t *FSTree) Get(addr *addressSDK.Address) ([]byte, error) { p := t.treePath(addr) if _, err := os.Stat(p); os.IsNotExist(err) { return nil, ErrFileNotFound } return os.ReadFile(p) } // NumberOfObjects walks the file tree rooted at FSTree's root // and returns number of stored objects. func (t *FSTree) NumberOfObjects() (uint64, error) { var counter uint64 // it is simpler to just consider every file // that is not directory as an object err := filepath.WalkDir(t.RootPath, func(_ string, d fs.DirEntry, _ error) error { if !d.IsDir() { counter++ } return nil }, ) if err != nil { return 0, fmt.Errorf("could not walk through %s directory: %w", t.RootPath, err) } return counter, nil }