[#472] blobstor: move fsTree to a separate package

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2021-04-08 16:53:25 +03:00 committed by Alex Vanin
parent a8a9f88d90
commit 934e394e28
9 changed files with 129 additions and 129 deletions

View file

@ -6,6 +6,7 @@ import (
"path" "path"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-node/pkg/util/logger" "github.com/nspcc-dev/neofs-node/pkg/util/logger"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -17,11 +18,13 @@ type BlobStor struct {
blobovniczas *blobovniczas blobovniczas *blobovniczas
} }
type Info = fstree.Info
// Option represents BlobStor's constructor option. // Option represents BlobStor's constructor option.
type Option func(*cfg) type Option func(*cfg)
type cfg struct { type cfg struct {
fsTree fsTree fsTree fstree.FSTree
compressor func([]byte) []byte compressor func([]byte) []byte
@ -54,9 +57,9 @@ const blobovniczaDir = "blobovnicza"
func defaultCfg() *cfg { func defaultCfg() *cfg {
return &cfg{ return &cfg{
fsTree: fsTree{ fsTree: fstree.FSTree{
depth: defaultShallowDepth, Depth: defaultShallowDepth,
dirNameLen: hex.EncodedLen(dirNameLen), DirNameLen: hex.EncodedLen(fstree.DirNameLen),
Info: Info{ Info: Info{
Permissions: defaultPerm, Permissions: defaultPerm,
RootPath: "./", RootPath: "./",
@ -92,11 +95,11 @@ func New(opts ...Option) *BlobStor {
// Depth is reduced to maximum value in case of overflow. // Depth is reduced to maximum value in case of overflow.
func WithShallowDepth(depth int) Option { func WithShallowDepth(depth int) Option {
return func(c *cfg) { return func(c *cfg) {
if depth > maxDepth { if depth > fstree.MaxDepth {
depth = maxDepth depth = fstree.MaxDepth
} }
c.fsTree.depth = depth c.fsTree.Depth = depth
} }
} }

View file

@ -1,10 +1,8 @@
package blobstor package blobstor
import ( import (
"os"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -23,19 +21,10 @@ type DeleteBigRes struct{}
// //
// Returns ErrNotFound if there is no object to delete. // Returns ErrNotFound if there is no object to delete.
func (b *BlobStor) DeleteBig(prm *DeleteBigPrm) (*DeleteBigRes, error) { func (b *BlobStor) DeleteBig(prm *DeleteBigPrm) (*DeleteBigRes, error) {
err := b.fsTree.delete(prm.addr) err := b.fsTree.Delete(prm.addr)
if errors.Is(err, errFileNotFound) { if errors.Is(err, fstree.ErrFileNotFound) {
err = object.ErrNotFound err = object.ErrNotFound
} }
return nil, err return nil, err
} }
func (t *fsTree) delete(addr *objectSDK.Address) error {
p, err := t.exists(addr)
if err != nil {
return err
}
return os.Remove(p)
}

View file

@ -1,10 +1,8 @@
package blobstor package blobstor
import ( import (
"os"
"github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/object"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -48,8 +46,8 @@ func (b *BlobStor) Exists(prm *ExistsPrm) (*ExistsRes, error) {
// checks if object is presented in shallow dir. // checks if object is presented in shallow dir.
func (b *BlobStor) existsBig(addr *object.Address) (bool, error) { func (b *BlobStor) existsBig(addr *object.Address) (bool, error) {
_, err := b.fsTree.exists(addr) _, err := b.fsTree.Exists(addr)
if errors.Is(err, errFileNotFound) { if errors.Is(err, fstree.ErrFileNotFound) {
return false, nil return false, nil
} }
@ -61,14 +59,3 @@ func (b *BlobStor) existsSmall(addr *object.Address) (bool, error) {
// TODO: implement // TODO: implement
return false, nil return false, nil
} }
func (t *fsTree) exists(addr *objectSDK.Address) (string, error) {
p := t.treePath(addr)
_, err := os.Stat(p)
if os.IsNotExist(err) {
err = errFileNotFound
}
return p, err
}

View file

@ -1,46 +0,0 @@
package blobstor
import (
"crypto/sha256"
"encoding/hex"
"errors"
"path"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
)
type fsTree struct {
depth int
dirNameLen int
Info
}
const dirNameLen = 1 // in bytes
var maxDepth = (sha256.Size - 1) / dirNameLen
var errFileNotFound = errors.New("file not found")
func stringifyAddress(addr *objectSDK.Address) string {
h := sha256.Sum256([]byte(addr.String()))
return hex.EncodeToString(h[:])
}
func (t *fsTree) treePath(addr *objectSDK.Address) string {
sAddr := stringifyAddress(addr)
dirs := make([]string, 0, t.depth+1+1) // 1 for root, 1 for file
dirs = append(dirs, t.RootPath)
for i := 0; i < t.depth; i++ {
dirs = append(dirs, sAddr[:t.dirNameLen])
sAddr = sAddr[t.dirNameLen:]
}
dirs = append(dirs, sAddr)
return path.Join(dirs...)
}

View file

@ -0,0 +1,106 @@
package fstree
import (
"crypto/sha256"
"encoding/hex"
"errors"
"io/ioutil"
"os"
"path"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
)
// FSTree represents object storage as filesystem tree.
type FSTree struct {
Info
Depth int
DirNameLen int
}
// Info groups the information about file storage.
type Info struct {
// Permission bits of the root directory.
Permissions os.FileMode
// Full path to the root directory.
RootPath string
}
const (
// DirNameLen is how many bytes is used to group keys into directories.
DirNameLen = 1 // in bytes
// MaxDepth is maximum depth of nested directories.
MaxDepth = (sha256.Size - 1) / DirNameLen
)
// ErrFileNotFound is returned when file is missing.
var ErrFileNotFound = errors.New("file not found")
func stringifyAddress(addr *objectSDK.Address) string {
h := sha256.Sum256([]byte(addr.String()))
return hex.EncodeToString(h[:])
}
func (t *FSTree) treePath(addr *objectSDK.Address) string {
sAddr := stringifyAddress(addr)
dirs := make([]string, 0, t.Depth+1+1) // 1 for root, 1 for file
dirs = append(dirs, t.RootPath)
for i := 0; i < t.Depth; i++ {
dirs = append(dirs, sAddr[:t.DirNameLen])
sAddr = sAddr[t.DirNameLen:]
}
dirs = append(dirs, sAddr)
return path.Join(dirs...)
}
// Delete removes object with the specified address from storage.
func (t *FSTree) Delete(addr *objectSDK.Address) error {
p, err := t.Exists(addr)
if err != nil {
return err
}
return os.Remove(p)
}
// Exists returns path to file with object contents if it exists in storage
// and an error otherwise.
func (t *FSTree) Exists(addr *objectSDK.Address) (string, error) {
p := t.treePath(addr)
_, err := os.Stat(p)
if os.IsNotExist(err) {
err = ErrFileNotFound
}
return p, err
}
// Put puts object in storage.
func (t *FSTree) Put(addr *objectSDK.Address, data []byte) error {
p := t.treePath(addr)
if err := os.MkdirAll(path.Dir(p), t.Permissions); err != nil {
return err
}
return ioutil.WriteFile(p, data, t.Permissions)
}
// Get returns object from storage by address.
func (t *FSTree) Get(addr *objectSDK.Address) ([]byte, error) {
p := t.treePath(addr)
if _, err := os.Stat(p); os.IsNotExist(err) {
return nil, ErrFileNotFound
}
return ioutil.ReadFile(p)
}

View file

@ -1,11 +1,8 @@
package blobstor package blobstor
import ( import (
"io/ioutil"
"os"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -28,9 +25,9 @@ type GetBigRes struct {
// presented in shallow dir. // presented in shallow dir.
func (b *BlobStor) GetBig(prm *GetBigPrm) (*GetBigRes, error) { func (b *BlobStor) GetBig(prm *GetBigPrm) (*GetBigRes, error) {
// get compressed object data // get compressed object data
data, err := b.fsTree.get(prm.addr) data, err := b.fsTree.Get(prm.addr)
if err != nil { if err != nil {
if errors.Is(err, errFileNotFound) { if errors.Is(err, fstree.ErrFileNotFound) {
return nil, object.ErrNotFound return nil, object.ErrNotFound
} }
@ -54,13 +51,3 @@ func (b *BlobStor) GetBig(prm *GetBigPrm) (*GetBigRes, error) {
}, },
}, nil }, nil
} }
func (t *fsTree) get(addr *objectSDK.Address) ([]byte, error) {
p := t.treePath(addr)
if _, err := os.Stat(p); os.IsNotExist(err) {
return nil, errFileNotFound
}
return ioutil.ReadFile(p)
}

View file

@ -24,7 +24,7 @@ type GetRangeBigRes struct {
// Returns ErrRangeOutOfBounds if requested object range is out of bounds. // Returns ErrRangeOutOfBounds if requested object range is out of bounds.
func (b *BlobStor) GetRangeBig(prm *GetRangeBigPrm) (*GetRangeBigRes, error) { func (b *BlobStor) GetRangeBig(prm *GetRangeBigPrm) (*GetRangeBigRes, error) {
// get compressed object data // get compressed object data
data, err := b.fsTree.get(prm.addr) data, err := b.fsTree.Get(prm.addr)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "could not read object from fs tree") return nil, errors.Wrap(err, "could not read object from fs tree")
} }

View file

@ -1,19 +1,8 @@
package blobstor package blobstor
import ( import "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"os"
)
// Info groups the information about BlobStor.
type Info struct {
// Permission bits of the root directory.
Permissions os.FileMode
// Full path to the root directory.
RootPath string
}
// DumpInfo returns information about the BlobStor. // DumpInfo returns information about the BlobStor.
func (b *BlobStor) DumpInfo() Info { func (b *BlobStor) DumpInfo() fstree.Info {
return b.fsTree.Info return b.fsTree.Info
} }

View file

@ -1,11 +1,6 @@
package blobstor package blobstor
import ( import (
"io/ioutil"
"os"
"path"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -41,7 +36,7 @@ func (b *BlobStor) Put(prm *PutPrm) (*PutRes, error) {
if big { if big {
// save object in shallow dir // save object in shallow dir
return new(PutRes), b.fsTree.put(prm.obj.Address(), data) return new(PutRes), b.fsTree.Put(prm.obj.Address(), data)
} }
// save object in blobovnicza // save object in blobovnicza
@ -57,16 +52,6 @@ func (b *BlobStor) Put(prm *PutPrm) (*PutRes, error) {
}, nil }, nil
} }
func (t *fsTree) put(addr *objectSDK.Address, data []byte) error {
p := t.treePath(addr)
if err := os.MkdirAll(path.Dir(p), t.Permissions); err != nil {
return err
}
return ioutil.WriteFile(p, data, t.Permissions)
}
// checks if object is "big". // checks if object is "big".
func (b *BlobStor) isBig(data []byte) bool { func (b *BlobStor) isBig(data []byte) bool {
return uint64(len(data)) > b.smallSizeLimit return uint64(len(data)) > b.smallSizeLimit