dstepanov-yadro/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#472] blobstor: move fsTree to a separate package

Browse source 
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2021-04-08 16:53:25 +03:00 committed by Alex Vanin
parent a8a9f88d90
commit 934e394e28
9 changed files with 129 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
pkg/local_object_storage/blobstor/blobstor.go
View file

@ -6,6 +6,7 @@ import (
"path"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-node/pkg/util/logger"
"go.uber.org/zap"
)
@ -17,11 +18,13 @@ type BlobStor struct {
blobovniczas *blobovniczas
}
type Info = fstree.Info
// Option represents BlobStor's constructor option.
type Option func(*cfg)
type cfg struct {
fsTree fsTree
fsTree fstree.FSTree
compressor func([]byte) []byte
@ -54,9 +57,9 @@ const blobovniczaDir = "blobovnicza"
func defaultCfg() *cfg {
return &cfg{
fsTree: fsTree{
depth: defaultShallowDepth,
dirNameLen: hex.EncodedLen(dirNameLen),
fsTree: fstree.FSTree{
Depth: defaultShallowDepth,
DirNameLen: hex.EncodedLen(fstree.DirNameLen),
Info: Info{
Permissions: defaultPerm,
RootPath: "./",
@ -92,11 +95,11 @@ func New(opts ...Option) *BlobStor {
// Depth is reduced to maximum value in case of overflow.
func WithShallowDepth(depth int) Option {
return func(c *cfg) {
if depth > maxDepth {
depth = maxDepth
if depth > fstree.MaxDepth {
depth = fstree.MaxDepth
}
c.fsTree.depth = depth
c.fsTree.Depth = depth
}
}

17
pkg/local_object_storage/blobstor/delete_big.go
View file

@ -1,10 +1,8 @@
package blobstor
import (
"os"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/pkg/errors"
)
@ -23,19 +21,10 @@ type DeleteBigRes struct{}
//
// Returns ErrNotFound if there is no object to delete.
func (b *BlobStor) DeleteBig(prm *DeleteBigPrm) (*DeleteBigRes, error) {
err := b.fsTree.delete(prm.addr)
if errors.Is(err, errFileNotFound) {
err := b.fsTree.Delete(prm.addr)
if errors.Is(err, fstree.ErrFileNotFound) {
err = object.ErrNotFound
}
return nil, err
}
func (t *fsTree) delete(addr *objectSDK.Address) error {
p, err := t.exists(addr)
if err != nil {
return err
}
return os.Remove(p)
}

19
pkg/local_object_storage/blobstor/exists.go
View file

@ -1,10 +1,8 @@
package blobstor
import (
"os"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/pkg/errors"
)
@ -48,8 +46,8 @@ func (b *BlobStor) Exists(prm *ExistsPrm) (*ExistsRes, error) {
// checks if object is presented in shallow dir.
func (b *BlobStor) existsBig(addr *object.Address) (bool, error) {
_, err := b.fsTree.exists(addr)
if errors.Is(err, errFileNotFound) {
_, err := b.fsTree.Exists(addr)
if errors.Is(err, fstree.ErrFileNotFound) {
return false, nil
}
@ -61,14 +59,3 @@ func (b *BlobStor) existsSmall(addr *object.Address) (bool, error) {
// TODO: implement
return false, nil
}
func (t *fsTree) exists(addr *objectSDK.Address) (string, error) {
p := t.treePath(addr)
_, err := os.Stat(p)
if os.IsNotExist(err) {
err = errFileNotFound
}
return p, err
}

46
pkg/local_object_storage/blobstor/fstree.go
View file

@ -1,46 +0,0 @@
package blobstor
import (
"crypto/sha256"
"encoding/hex"
"errors"
"path"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
)
type fsTree struct {
depth int
dirNameLen int
Info
}
const dirNameLen = 1 // in bytes
var maxDepth = (sha256.Size - 1) / dirNameLen
var errFileNotFound = errors.New("file not found")
func stringifyAddress(addr *objectSDK.Address) string {
h := sha256.Sum256([]byte(addr.String()))
return hex.EncodeToString(h[:])
}
func (t *fsTree) treePath(addr *objectSDK.Address) string {
sAddr := stringifyAddress(addr)
dirs := make([]string, 0, t.depth+1+1) // 1 for root, 1 for file
dirs = append(dirs, t.RootPath)
for i := 0; i < t.depth; i++ {
dirs = append(dirs, sAddr[:t.dirNameLen])
sAddr = sAddr[t.dirNameLen:]
}
dirs = append(dirs, sAddr)
return path.Join(dirs...)
}

106
pkg/local_object_storage/blobstor/fstree/fstree.go Normal file
View file

@ -0,0 +1,106 @@
package fstree
import (
"crypto/sha256"
"encoding/hex"
"errors"
"io/ioutil"
"os"
"path"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
)
// FSTree represents object storage as filesystem tree.
type FSTree struct {
Info
Depth int
DirNameLen int
}
// Info groups the information about file storage.
type Info struct {
// Permission bits of the root directory.
Permissions os.FileMode
// Full path to the root directory.
RootPath string
}
const (
// DirNameLen is how many bytes is used to group keys into directories.
DirNameLen = 1 // in bytes
// MaxDepth is maximum depth of nested directories.
MaxDepth = (sha256.Size - 1) / DirNameLen
)
// ErrFileNotFound is returned when file is missing.
var ErrFileNotFound = errors.New("file not found")
func stringifyAddress(addr *objectSDK.Address) string {
h := sha256.Sum256([]byte(addr.String()))
return hex.EncodeToString(h[:])
}
func (t *FSTree) treePath(addr *objectSDK.Address) string {
sAddr := stringifyAddress(addr)
dirs := make([]string, 0, t.Depth+1+1) // 1 for root, 1 for file
dirs = append(dirs, t.RootPath)
for i := 0; i < t.Depth; i++ {
dirs = append(dirs, sAddr[:t.DirNameLen])
sAddr = sAddr[t.DirNameLen:]
}
dirs = append(dirs, sAddr)
return path.Join(dirs...)
}
// Delete removes object with the specified address from storage.
func (t *FSTree) Delete(addr *objectSDK.Address) error {
p, err := t.Exists(addr)
if err != nil {
return err
}
return os.Remove(p)
}
// Exists returns path to file with object contents if it exists in storage
// and an error otherwise.
func (t *FSTree) Exists(addr *objectSDK.Address) (string, error) {
p := t.treePath(addr)
_, err := os.Stat(p)
if os.IsNotExist(err) {
err = ErrFileNotFound
}
return p, err
}
// Put puts object in storage.
func (t *FSTree) Put(addr *objectSDK.Address, data []byte) error {
p := t.treePath(addr)
if err := os.MkdirAll(path.Dir(p), t.Permissions); err != nil {
return err
}
return ioutil.WriteFile(p, data, t.Permissions)
}
// Get returns object from storage by address.
func (t *FSTree) Get(addr *objectSDK.Address) ([]byte, error) {
p := t.treePath(addr)
if _, err := os.Stat(p); os.IsNotExist(err) {
return nil, ErrFileNotFound
}
return ioutil.ReadFile(p)
}

19
pkg/local_object_storage/blobstor/get_big.go
View file

@ -1,11 +1,8 @@
package blobstor
import (
"io/ioutil"
"os"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/pkg/errors"
)
@ -28,9 +25,9 @@ type GetBigRes struct {
// presented in shallow dir.
func (b *BlobStor) GetBig(prm *GetBigPrm) (*GetBigRes, error) {
// get compressed object data
data, err := b.fsTree.get(prm.addr)
data, err := b.fsTree.Get(prm.addr)
if err != nil {
if errors.Is(err, errFileNotFound) {
if errors.Is(err, fstree.ErrFileNotFound) {
return nil, object.ErrNotFound
}
@ -54,13 +51,3 @@ func (b *BlobStor) GetBig(prm *GetBigPrm) (*GetBigRes, error) {
},
}, nil
}
func (t *fsTree) get(addr *objectSDK.Address) ([]byte, error) {
p := t.treePath(addr)
if _, err := os.Stat(p); os.IsNotExist(err) {
return nil, errFileNotFound
}
return ioutil.ReadFile(p)
}

2
pkg/local_object_storage/blobstor/get_range_big.go
View file

@ -24,7 +24,7 @@ type GetRangeBigRes struct {
// Returns ErrRangeOutOfBounds if requested object range is out of bounds.
func (b *BlobStor) GetRangeBig(prm *GetRangeBigPrm) (*GetRangeBigRes, error) {
// get compressed object data
data, err := b.fsTree.get(prm.addr)
data, err := b.fsTree.Get(prm.addr)
if err != nil {
return nil, errors.Wrap(err, "could not read object from fs tree")
}

15
pkg/local_object_storage/blobstor/info.go
View file

@ -1,19 +1,8 @@
package blobstor
import (
"os"
)
// Info groups the information about BlobStor.
type Info struct {
// Permission bits of the root directory.
Permissions os.FileMode
// Full path to the root directory.
RootPath string
}
import "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
// DumpInfo returns information about the BlobStor.
func (b *BlobStor) DumpInfo() Info {
func (b *BlobStor) DumpInfo() fstree.Info {
return b.fsTree.Info
}

17
pkg/local_object_storage/blobstor/put.go
View file

@ -1,11 +1,6 @@
package blobstor
import (
"io/ioutil"
"os"
"path"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/pkg/errors"
)
@ -41,7 +36,7 @@ func (b *BlobStor) Put(prm *PutPrm) (*PutRes, error) {
if big {
// save object in shallow dir
return new(PutRes), b.fsTree.put(prm.obj.Address(), data)
return new(PutRes), b.fsTree.Put(prm.obj.Address(), data)
}
// save object in blobovnicza
@ -57,16 +52,6 @@ func (b *BlobStor) Put(prm *PutPrm) (*PutRes, error) {
}, nil
}
func (t *fsTree) put(addr *objectSDK.Address, data []byte) error {
p := t.treePath(addr)
if err := os.MkdirAll(path.Dir(p), t.Permissions); err != nil {
return err
}
return ioutil.WriteFile(p, data, t.Permissions)
}
// checks if object is "big".
func (b *BlobStor) isBig(data []byte) bool {
return uint64(len(data)) > b.smallSizeLimit