From 934e394e2883a0aa1fbe8a1470083c7e3c583549 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Apr 2021 16:53:25 +0300 Subject: [PATCH] [#472] blobstor: move fsTree to a separate package Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobstor/blobstor.go | 17 +-- .../blobstor/delete_big.go | 17 +-- pkg/local_object_storage/blobstor/exists.go | 19 +--- pkg/local_object_storage/blobstor/fstree.go | 46 -------- .../blobstor/fstree/fstree.go | 106 ++++++++++++++++++ pkg/local_object_storage/blobstor/get_big.go | 19 +--- .../blobstor/get_range_big.go | 2 +- pkg/local_object_storage/blobstor/info.go | 15 +-- pkg/local_object_storage/blobstor/put.go | 17 +-- 9 files changed, 129 insertions(+), 129 deletions(-) delete mode 100644 pkg/local_object_storage/blobstor/fstree.go create mode 100644 pkg/local_object_storage/blobstor/fstree/fstree.go diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index b08c3f08f..aa3fdddcf 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -6,6 +6,7 @@ import ( "path" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" "github.com/nspcc-dev/neofs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -17,11 +18,13 @@ type BlobStor struct { blobovniczas *blobovniczas } +type Info = fstree.Info + // Option represents BlobStor's constructor option. type Option func(*cfg) type cfg struct { - fsTree fsTree + fsTree fstree.FSTree compressor func([]byte) []byte @@ -54,9 +57,9 @@ const blobovniczaDir = "blobovnicza" func defaultCfg() *cfg { return &cfg{ - fsTree: fsTree{ - depth: defaultShallowDepth, - dirNameLen: hex.EncodedLen(dirNameLen), + fsTree: fstree.FSTree{ + Depth: defaultShallowDepth, + DirNameLen: hex.EncodedLen(fstree.DirNameLen), Info: Info{ Permissions: defaultPerm, RootPath: "./", @@ -92,11 +95,11 @@ func New(opts ...Option) *BlobStor { // Depth is reduced to maximum value in case of overflow. func WithShallowDepth(depth int) Option { return func(c *cfg) { - if depth > maxDepth { - depth = maxDepth + if depth > fstree.MaxDepth { + depth = fstree.MaxDepth } - c.fsTree.depth = depth + c.fsTree.Depth = depth } } diff --git a/pkg/local_object_storage/blobstor/delete_big.go b/pkg/local_object_storage/blobstor/delete_big.go index 85e3fa94b..b1565a8d1 100644 --- a/pkg/local_object_storage/blobstor/delete_big.go +++ b/pkg/local_object_storage/blobstor/delete_big.go @@ -1,10 +1,8 @@ package blobstor import ( - "os" - - objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-node/pkg/core/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" "github.com/pkg/errors" ) @@ -23,19 +21,10 @@ type DeleteBigRes struct{} // // Returns ErrNotFound if there is no object to delete. func (b *BlobStor) DeleteBig(prm *DeleteBigPrm) (*DeleteBigRes, error) { - err := b.fsTree.delete(prm.addr) - if errors.Is(err, errFileNotFound) { + err := b.fsTree.Delete(prm.addr) + if errors.Is(err, fstree.ErrFileNotFound) { err = object.ErrNotFound } return nil, err } - -func (t *fsTree) delete(addr *objectSDK.Address) error { - p, err := t.exists(addr) - if err != nil { - return err - } - - return os.Remove(p) -} diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index c20b21c1c..5240accfd 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -1,10 +1,8 @@ package blobstor import ( - "os" - "github.com/nspcc-dev/neofs-api-go/pkg/object" - objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" "github.com/pkg/errors" ) @@ -48,8 +46,8 @@ func (b *BlobStor) Exists(prm *ExistsPrm) (*ExistsRes, error) { // checks if object is presented in shallow dir. func (b *BlobStor) existsBig(addr *object.Address) (bool, error) { - _, err := b.fsTree.exists(addr) - if errors.Is(err, errFileNotFound) { + _, err := b.fsTree.Exists(addr) + if errors.Is(err, fstree.ErrFileNotFound) { return false, nil } @@ -61,14 +59,3 @@ func (b *BlobStor) existsSmall(addr *object.Address) (bool, error) { // TODO: implement return false, nil } - -func (t *fsTree) exists(addr *objectSDK.Address) (string, error) { - p := t.treePath(addr) - - _, err := os.Stat(p) - if os.IsNotExist(err) { - err = errFileNotFound - } - - return p, err -} diff --git a/pkg/local_object_storage/blobstor/fstree.go b/pkg/local_object_storage/blobstor/fstree.go deleted file mode 100644 index 4dabc80a5..000000000 --- a/pkg/local_object_storage/blobstor/fstree.go +++ /dev/null @@ -1,46 +0,0 @@ -package blobstor - -import ( - "crypto/sha256" - "encoding/hex" - "errors" - "path" - - objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" -) - -type fsTree struct { - depth int - - dirNameLen int - - Info -} - -const dirNameLen = 1 // in bytes - -var maxDepth = (sha256.Size - 1) / dirNameLen - -var errFileNotFound = errors.New("file not found") - -func stringifyAddress(addr *objectSDK.Address) string { - h := sha256.Sum256([]byte(addr.String())) - - return hex.EncodeToString(h[:]) -} - -func (t *fsTree) treePath(addr *objectSDK.Address) string { - sAddr := stringifyAddress(addr) - - dirs := make([]string, 0, t.depth+1+1) // 1 for root, 1 for file - dirs = append(dirs, t.RootPath) - - for i := 0; i < t.depth; i++ { - dirs = append(dirs, sAddr[:t.dirNameLen]) - sAddr = sAddr[t.dirNameLen:] - } - - dirs = append(dirs, sAddr) - - return path.Join(dirs...) -} diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go new file mode 100644 index 000000000..bcdabb87e --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -0,0 +1,106 @@ +package fstree + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "io/ioutil" + "os" + "path" + + objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" +) + +// FSTree represents object storage as filesystem tree. +type FSTree struct { + Info + + Depth int + DirNameLen int +} + +// Info groups the information about file storage. +type Info struct { + // Permission bits of the root directory. + Permissions os.FileMode + + // Full path to the root directory. + RootPath string +} + +const ( + // DirNameLen is how many bytes is used to group keys into directories. + DirNameLen = 1 // in bytes + // MaxDepth is maximum depth of nested directories. + MaxDepth = (sha256.Size - 1) / DirNameLen +) + +// ErrFileNotFound is returned when file is missing. +var ErrFileNotFound = errors.New("file not found") + +func stringifyAddress(addr *objectSDK.Address) string { + h := sha256.Sum256([]byte(addr.String())) + + return hex.EncodeToString(h[:]) +} + +func (t *FSTree) treePath(addr *objectSDK.Address) string { + sAddr := stringifyAddress(addr) + + dirs := make([]string, 0, t.Depth+1+1) // 1 for root, 1 for file + dirs = append(dirs, t.RootPath) + + for i := 0; i < t.Depth; i++ { + dirs = append(dirs, sAddr[:t.DirNameLen]) + sAddr = sAddr[t.DirNameLen:] + } + + dirs = append(dirs, sAddr) + + return path.Join(dirs...) +} + +// Delete removes object with the specified address from storage. +func (t *FSTree) Delete(addr *objectSDK.Address) error { + p, err := t.Exists(addr) + if err != nil { + return err + } + + return os.Remove(p) +} + +// Exists returns path to file with object contents if it exists in storage +// and an error otherwise. +func (t *FSTree) Exists(addr *objectSDK.Address) (string, error) { + p := t.treePath(addr) + + _, err := os.Stat(p) + if os.IsNotExist(err) { + err = ErrFileNotFound + } + + return p, err +} + +// Put puts object in storage. +func (t *FSTree) Put(addr *objectSDK.Address, data []byte) error { + p := t.treePath(addr) + + if err := os.MkdirAll(path.Dir(p), t.Permissions); err != nil { + return err + } + + return ioutil.WriteFile(p, data, t.Permissions) +} + +// Get returns object from storage by address. +func (t *FSTree) Get(addr *objectSDK.Address) ([]byte, error) { + p := t.treePath(addr) + + if _, err := os.Stat(p); os.IsNotExist(err) { + return nil, ErrFileNotFound + } + + return ioutil.ReadFile(p) +} diff --git a/pkg/local_object_storage/blobstor/get_big.go b/pkg/local_object_storage/blobstor/get_big.go index 9551a0cd5..418f0d9b3 100644 --- a/pkg/local_object_storage/blobstor/get_big.go +++ b/pkg/local_object_storage/blobstor/get_big.go @@ -1,11 +1,8 @@ package blobstor import ( - "io/ioutil" - "os" - - objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-node/pkg/core/object" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" "github.com/pkg/errors" ) @@ -28,9 +25,9 @@ type GetBigRes struct { // presented in shallow dir. func (b *BlobStor) GetBig(prm *GetBigPrm) (*GetBigRes, error) { // get compressed object data - data, err := b.fsTree.get(prm.addr) + data, err := b.fsTree.Get(prm.addr) if err != nil { - if errors.Is(err, errFileNotFound) { + if errors.Is(err, fstree.ErrFileNotFound) { return nil, object.ErrNotFound } @@ -54,13 +51,3 @@ func (b *BlobStor) GetBig(prm *GetBigPrm) (*GetBigRes, error) { }, }, nil } - -func (t *fsTree) get(addr *objectSDK.Address) ([]byte, error) { - p := t.treePath(addr) - - if _, err := os.Stat(p); os.IsNotExist(err) { - return nil, errFileNotFound - } - - return ioutil.ReadFile(p) -} diff --git a/pkg/local_object_storage/blobstor/get_range_big.go b/pkg/local_object_storage/blobstor/get_range_big.go index 4bb45336d..bcdfabd33 100644 --- a/pkg/local_object_storage/blobstor/get_range_big.go +++ b/pkg/local_object_storage/blobstor/get_range_big.go @@ -24,7 +24,7 @@ type GetRangeBigRes struct { // Returns ErrRangeOutOfBounds if requested object range is out of bounds. func (b *BlobStor) GetRangeBig(prm *GetRangeBigPrm) (*GetRangeBigRes, error) { // get compressed object data - data, err := b.fsTree.get(prm.addr) + data, err := b.fsTree.Get(prm.addr) if err != nil { return nil, errors.Wrap(err, "could not read object from fs tree") } diff --git a/pkg/local_object_storage/blobstor/info.go b/pkg/local_object_storage/blobstor/info.go index c52901788..3e8b04339 100644 --- a/pkg/local_object_storage/blobstor/info.go +++ b/pkg/local_object_storage/blobstor/info.go @@ -1,19 +1,8 @@ package blobstor -import ( - "os" -) - -// Info groups the information about BlobStor. -type Info struct { - // Permission bits of the root directory. - Permissions os.FileMode - - // Full path to the root directory. - RootPath string -} +import "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" // DumpInfo returns information about the BlobStor. -func (b *BlobStor) DumpInfo() Info { +func (b *BlobStor) DumpInfo() fstree.Info { return b.fsTree.Info } diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index 4d83e8972..8ecc9773b 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -1,11 +1,6 @@ package blobstor import ( - "io/ioutil" - "os" - "path" - - objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/pkg/errors" ) @@ -41,7 +36,7 @@ func (b *BlobStor) Put(prm *PutPrm) (*PutRes, error) { if big { // save object in shallow dir - return new(PutRes), b.fsTree.put(prm.obj.Address(), data) + return new(PutRes), b.fsTree.Put(prm.obj.Address(), data) } // save object in blobovnicza @@ -57,16 +52,6 @@ func (b *BlobStor) Put(prm *PutPrm) (*PutRes, error) { }, nil } -func (t *fsTree) put(addr *objectSDK.Address, data []byte) error { - p := t.treePath(addr) - - if err := os.MkdirAll(path.Dir(p), t.Permissions); err != nil { - return err - } - - return ioutil.WriteFile(p, data, t.Permissions) -} - // checks if object is "big". func (b *BlobStor) isBig(data []byte) bool { return uint64(len(data)) > b.smallSizeLimit