TrueCloudLab/frostfs-node
21
1
Fork 27
Code Issues 108 Pull requests 4 Projects Releases 2 Activity Actions

[#645] blobtree: Store path as storageID

Browse source 
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-10-09 11:45:42 +03:00
parent 2f6d4a49dc
commit 4daef70774
10 changed files with 106 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
pkg/local_object_storage/blobstor/blobtree/blobtree.go
View file

@ -48,27 +48,29 @@ func New(opts ...Option) *BlobTree {
return b
}
func (b *BlobTree) getDirectoryPath(addr oid.Address) string {
func (b *BlobTree) getDir(addr oid.Address) string {
sAddr := addr.Object().EncodeToString() + "." + addr.Container().EncodeToString()
var sb strings.Builder
size := int(1+b.cfg.depth*(directoryLength+1)) + len(b.cfg.rootPath) // /path + slash + (character + slash for every level)
size := int(b.cfg.depth * (directoryLength + 1)) // (character + slash for every level)
sb.Grow(size)
sb.WriteString(b.cfg.rootPath)
for i := uint64(0); i < b.cfg.depth; i++ {
sb.WriteRune(filepath.Separator)
if i > 0 {
sb.WriteRune(filepath.Separator)
}
sb.WriteString(sAddr[:directoryLength])
sAddr = sAddr[directoryLength:]
}
sb.WriteRune(filepath.Separator)
return sb.String()
}
func (b *BlobTree) createDir(dir string) error {
func (b *BlobTree) createDir(dir string, isSystemPath bool) error {
b.dirLock.Lock(dir)
defer b.dirLock.Unlock(dir)
if !isSystemPath {
dir = b.getSystemPath(dir)
}
if err := util.MkdirAllX(dir, b.cfg.permissions); err != nil {
if errors.Is(err, syscall.ENOSPC) {
err = common.ErrNoSpace

18
pkg/local_object_storage/blobstor/blobtree/content.go
View file

@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"strconv"
"strings"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -41,7 +42,7 @@ type objectData struct {
}
func (b *BlobTree) readFileContent(path string) ([]objectData, error) {
rawData, err := os.ReadFile(path)
rawData, err := os.ReadFile(b.getSystemPath(path))
if err != nil {
if os.IsNotExist(err) {
return []objectData{}, nil
@ -130,6 +131,7 @@ func (b *BlobTree) saveContentToFile(records []objectData, path string) (uint64,
}
func (b *BlobTree) writeFile(p string, data []byte) error {
p = b.getSystemPath(p)
f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_SYNC, b.cfg.permissions)
if err != nil {
return err
@ -141,6 +143,10 @@ func (b *BlobTree) writeFile(p string, data []byte) error {
return err
}
func (b *BlobTree) getSystemPath(path string) string {
return filepath.Join(b.cfg.rootPath, path)
}
func (b *BlobTree) marshalSlice(records []objectData) ([]byte, error) {
buf := make([]byte, b.estimateSize(records))
result := buf
@ -191,3 +197,13 @@ func (b *BlobTree) estimateSize(records []objectData) uint64 {
func (b *BlobTree) getFilePath(dir string, idx uint64) string {
return filepath.Join(dir, strconv.FormatUint(idx, 16)+dataExtension)
}
func (b *BlobTree) parsePath(path string) (string, uint64, error) {
dir := filepath.Dir(path)
fileName := strings.TrimSuffix(filepath.Base(path), dataExtension)
idx, err := strconv.ParseUint(fileName, 16, 64)
if err != nil {
return "", 0, fmt.Errorf("failed to parse blobtree path: %w", err)
}
return dir, idx, nil
}

22
pkg/local_object_storage/blobstor/blobtree/control.go
View file

@ -19,37 +19,35 @@ func (b *BlobTree) Open(readOnly bool) error {
}
func (b *BlobTree) Init() error {
if err := b.createDir(b.cfg.rootPath); err != nil {
if err := b.createDir(b.cfg.rootPath, true); err != nil {
return err
}
var eg errgroup.Group
eg.SetLimit(b.cfg.initWorkersCount)
eg.Go(func() error {
return b.initDir(&eg, b.cfg.rootPath, 0)
return b.initDir(&eg, "")
})
return eg.Wait()
}
func (b *BlobTree) initDir(eg *errgroup.Group, dir string, depth uint64) error {
entities, err := os.ReadDir(dir)
func (b *BlobTree) initDir(eg *errgroup.Group, dir string) error {
entities, err := os.ReadDir(b.getSystemPath(dir))
if err != nil {
return err
}
for _, entity := range entities {
if depth < b.cfg.depth && entity.IsDir() {
if entity.IsDir() {
eg.Go(func() error {
return b.initDir(eg, filepath.Join(dir, entity.Name()), depth+1)
return b.initDir(eg, filepath.Join(dir, entity.Name()))
})
continue
}
if depth != b.cfg.depth {
continue
}
path := filepath.Join(dir, entity.Name())
if b.isTempFile(entity.Name()) {
if err = os.Remove(filepath.Join(dir, entity.Name())); err != nil {
if err = os.Remove(b.getSystemPath(path)); err != nil {
return err
}
continue
@ -57,12 +55,12 @@ func (b *BlobTree) initDir(eg *errgroup.Group, dir string, depth uint64) error {
idx, err := b.parseIdx(entity.Name())
if err != nil {
continue
return err
}
b.dispatcher.Init(dir, idx)
b.cfg.metrics.IncFilesCount()
stat, err := os.Stat(filepath.Join(dir, entity.Name()))
stat, err := os.Stat(b.getSystemPath(path))
if err != nil {
return err
}

27
pkg/local_object_storage/blobstor/blobtree/delete.go
View file

@ -3,6 +3,7 @@ package blobtree
import (
"context"
"os"
"path/filepath"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
@ -27,7 +28,7 @@ func (b *BlobTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Del
trace.WithAttributes(
attribute.String("path", b.cfg.rootPath),
attribute.String("address", prm.Address.EncodeToString()),
attribute.String("storage_id", storageIDToIdxStringSafe(prm.StorageID)),
attribute.String("storage_id", string(prm.StorageID)),
))
defer span.End()
@ -37,8 +38,8 @@ func (b *BlobTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Del
var res common.DeleteRes
var err error
if idx, ok := tryParseIdxFromStorageID(prm.StorageID); ok {
res, err = b.deleteFromIdx(prm.Address, idx)
if path, ok := getPathFromStorageID(prm.StorageID); ok {
res, err = b.deleteFromPath(prm.Address, path)
} else {
res, err = b.findAndDelete(prm.Address)
}
@ -46,13 +47,15 @@ func (b *BlobTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Del
return res, err
}
func (b *BlobTree) deleteFromIdx(addr oid.Address, idx uint64) (common.DeleteRes, error) {
dir := b.getDirectoryPath(addr)
path := b.getFilePath(dir, idx)
func (b *BlobTree) deleteFromPath(addr oid.Address, path string) (common.DeleteRes, error) {
b.fileLock.Lock(path)
defer b.fileLock.Unlock(path)
dir, idx, err := b.parsePath(path)
if err != nil {
return common.DeleteRes{}, err
}
records, err := b.readFileContent(path)
if err != nil {
return common.DeleteRes{}, err
@ -71,7 +74,7 @@ func (b *BlobTree) deleteFromIdx(addr oid.Address, idx uint64) (common.DeleteRes
}
if len(records) == 1 {
err = os.Remove(path)
err = os.Remove(b.getSystemPath(path))
if err == nil {
b.dispatcher.ReturnIdx(dir, idx)
b.cfg.metrics.DecFilesCount()
@ -79,7 +82,7 @@ func (b *BlobTree) deleteFromIdx(addr oid.Address, idx uint64) (common.DeleteRes
return common.DeleteRes{}, err
}
records = append(records[:idx], records[idx+1:]...)
records = append(records[:deleteIdx], records[deleteIdx+1:]...)
size, err := b.writeToTmpAndRename(records, path)
if err != nil {
return common.DeleteRes{}, err
@ -91,16 +94,16 @@ func (b *BlobTree) deleteFromIdx(addr oid.Address, idx uint64) (common.DeleteRes
}
func (b *BlobTree) findAndDelete(addr oid.Address) (common.DeleteRes, error) {
dir := b.getDirectoryPath(addr)
dir := b.getDir(addr)
idx, err := b.findFileIdx(dir, addr)
if err != nil {
return common.DeleteRes{}, err
}
return b.deleteFromIdx(addr, idx)
return b.deleteFromPath(addr, b.getFilePath(dir, idx))
}
func (b *BlobTree) findFileIdx(dir string, addr oid.Address) (uint64, error) {
entities, err := os.ReadDir(dir)
entities, err := os.ReadDir(filepath.Join(b.cfg.rootPath, dir))
if err != nil {
if os.IsNotExist(err) {
return 0, logicerr.Wrap(new(apistatus.ObjectNotFound))

32
pkg/local_object_storage/blobstor/blobtree/exists.go
View file

@ -26,25 +26,22 @@ func (b *BlobTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi
trace.WithAttributes(
attribute.String("path", b.cfg.rootPath),
attribute.String("address", prm.Address.EncodeToString()),
attribute.String("storage_id", storageIDToIdxStringSafe(prm.StorageID)),
attribute.String("storage_id", string(prm.StorageID)),
))
defer span.End()
var res common.ExistsRes
var err error
if idx, ok := tryParseIdxFromStorageID(prm.StorageID); ok {
res, err = b.existsFromIdx(prm.Address, idx)
if path, ok := getPathFromStorageID(prm.StorageID); ok {
res, err = b.existsFromPath(prm.Address, path)
} else {
res, err = b.findAndCheck(prm.Address)
res, err = b.findAndCheckExistance(prm.Address)
}
success = err == nil
return res, err
}
func (b *BlobTree) existsFromIdx(addr oid.Address, idx uint64) (common.ExistsRes, error) {
dir := b.getDirectoryPath(addr)
path := b.getFilePath(dir, idx)
func (b *BlobTree) existsFromPath(addr oid.Address, path string) (common.ExistsRes, error) {
b.fileLock.RLock(path)
defer b.fileLock.RUnlock(path)
@ -64,15 +61,16 @@ func (b *BlobTree) existsFromIdx(addr oid.Address, idx uint64) (common.ExistsRes
return common.ExistsRes{}, nil
}
func (b *BlobTree) findAndCheck(addr oid.Address) (common.ExistsRes, error) {
dir := b.getDirectoryPath(addr)
func (b *BlobTree) findAndCheckExistance(addr oid.Address) (common.ExistsRes, error) {
dir := b.getDir(addr)
_, err := b.findFileIdx(dir, addr)
if err != nil {
var notFound *apistatus.ObjectNotFound
if errors.As(err, &notFound) {
return common.ExistsRes{}, nil
}
return common.ExistsRes{}, err
if err == nil {
return common.ExistsRes{Exists: true}, nil
}
return common.ExistsRes{Exists: true}, nil
var notFound *apistatus.ObjectNotFound
if errors.As(err, &notFound) {
return common.ExistsRes{}, nil
}
return common.ExistsRes{}, err
}

22
pkg/local_object_storage/blobstor/blobtree/get.go
View file

@ -3,6 +3,7 @@ package blobtree
import (
"context"
"os"
"path/filepath"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
@ -29,7 +30,7 @@ func (b *BlobTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, e
trace.WithAttributes(
attribute.String("path", b.cfg.rootPath),
attribute.String("address", prm.Address.EncodeToString()),
attribute.String("storage_id", storageIDToIdxStringSafe(prm.StorageID)),
attribute.String("storage_id", string(prm.StorageID)),
attribute.Bool("raw", prm.Raw),
))
defer span.End()
@ -41,16 +42,13 @@ func (b *BlobTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, e
}
func (b *BlobTree) get(prm common.GetPrm) (common.GetRes, error) {
if idx, ok := tryParseIdxFromStorageID(prm.StorageID); ok {
return b.getFromIdx(prm.Address, idx)
if path, ok := getPathFromStorageID(prm.StorageID); ok {
return b.getFromPath(prm.Address, path)
}
return b.findAndGet(prm.Address)
}
func (b *BlobTree) getFromIdx(addr oid.Address, idx uint64) (common.GetRes, error) {
dir := b.getDirectoryPath(addr)
path := b.getFilePath(dir, idx)
func (b *BlobTree) getFromPath(addr oid.Address, path string) (common.GetRes, error) {
b.fileLock.RLock(path)
defer b.fileLock.RUnlock(path)
@ -82,8 +80,8 @@ func (b *BlobTree) unmarshalGetRes(record objectData) (common.GetRes, error) {
}
func (b *BlobTree) findAndGet(addr oid.Address) (common.GetRes, error) {
dir := b.getDirectoryPath(addr)
entities, err := os.ReadDir(dir)
dir := b.getDir(addr)
entities, err := os.ReadDir(filepath.Join(b.cfg.rootPath, dir))
if err != nil {
if os.IsNotExist(err) {
return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound))
@ -97,11 +95,7 @@ func (b *BlobTree) findAndGet(addr oid.Address) (common.GetRes, error) {
if b.isTempFile(entity.Name()) {
continue
}
idx, err := b.parseIdx(entity.Name())
if err != nil {
continue
}
path := b.getFilePath(dir, idx)
path := filepath.Join(dir, entity.Name())
res, err := b.tryReadObject(path, addr)
if err != nil {
return common.GetRes{}, err

2
pkg/local_object_storage/blobstor/blobtree/get_range.go
View file

@ -27,7 +27,7 @@ func (b *BlobTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common
trace.WithAttributes(
attribute.String("path", b.cfg.rootPath),
attribute.String("address", prm.Address.EncodeToString()),
attribute.String("storage_id", storageIDToIdxStringSafe(prm.StorageID)),
attribute.String("storage_id", string(prm.StorageID)),
attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)),
attribute.String("length", strconv.FormatUint(prm.Range.GetLength(), 10)),
))

26
pkg/local_object_storage/blobstor/blobtree/iterate.go
View file

@ -28,12 +28,12 @@ func (b *BlobTree) Iterate(ctx context.Context, prm common.IteratePrm) (common.I
))
defer span.End()
err = b.iterateDir(b.cfg.rootPath, 0, prm)
err = b.iterateDir("", prm)
return common.IterateRes{}, err
}
func (b *BlobTree) iterateDir(dir string, depth uint64, prm common.IteratePrm) error {
entities, err := os.ReadDir(dir)
func (b *BlobTree) iterateDir(dir string, prm common.IteratePrm) error {
entities, err := os.ReadDir(filepath.Join(b.cfg.rootPath, dir))
if err != nil {
if prm.IgnoreErrors {
return nil
@ -42,24 +42,20 @@ func (b *BlobTree) iterateDir(dir string, depth uint64, prm common.IteratePrm) e
}
for _, entity := range entities {
if depth < b.cfg.depth && entity.IsDir() {
err := b.iterateDir(filepath.Join(dir, entity.Name()), depth+1, prm)
if entity.IsDir() {
err := b.iterateDir(filepath.Join(dir, entity.Name()), prm)
if err != nil {
return err
}
}
if depth != b.cfg.depth {
continue
}
if b.isTempFile(entity.Name()) {
continue
}
idx, err := b.parseIdx(entity.Name())
if err != nil {
continue
}
path := b.getFilePath(dir, idx)
err = b.iterateRecords(idx, path, prm)
path := filepath.Join(dir, entity.Name())
err = b.iterateRecords(path, prm)
if err != nil {
return err
}
@ -67,7 +63,7 @@ func (b *BlobTree) iterateDir(dir string, depth uint64, prm common.IteratePrm) e
return nil
}
func (b *BlobTree) iterateRecords(idx uint64, path string, prm common.IteratePrm) error {
func (b *BlobTree) iterateRecords(path string, prm common.IteratePrm) error {
b.fileLock.RLock(path)
defer b.fileLock.RUnlock(path)
@ -103,7 +99,7 @@ func (b *BlobTree) iterateRecords(idx uint64, path string, prm common.IteratePrm
err = prm.Handler(common.IterationElement{
Address: record.Address,
ObjectData: record.Data,
StorageID: idxToStorageID(idx),
StorageID: pathToStorageID(path),
})
if err != nil {
return err

28
pkg/local_object_storage/blobstor/blobtree/put.go
View file

@ -37,9 +37,9 @@ func (b *BlobTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e
return common.PutRes{}, common.ErrReadOnly
}
dir := b.getDirectoryPath(prm.Address)
dir := b.getDir(prm.Address)
if err := b.createDir(dir); err != nil {
if err := b.createDir(dir, false); err != nil {
return common.PutRes{}, err
}
@ -47,7 +47,7 @@ func (b *BlobTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e
prm.RawData = b.compressor.Compress(prm.RawData)
}
idx, err := b.saveToFile(prm, dir)
path, err := b.saveToLocalDir(prm, dir)
if err != nil {
return common.PutRes{}, err
}
@ -55,10 +55,10 @@ func (b *BlobTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e
success = true
size = len(prm.RawData)
return common.PutRes{StorageID: idxToStorageID(idx)}, nil
return common.PutRes{StorageID: pathToStorageID(path)}, nil
}
func (b *BlobTree) saveToFile(prm common.PutPrm, dir string) (uint64, error) {
func (b *BlobTree) saveToLocalDir(prm common.PutPrm, dir string) (string, error) {
returnIdx := true
idx := b.dispatcher.GetIdxForWrite(dir)
path := b.getFilePath(dir, idx)
@ -74,7 +74,7 @@ func (b *BlobTree) saveToFile(prm common.PutPrm, dir string) (uint64, error) {
currentContent, err := b.readFileContent(path)
if err != nil {
return 0, err
return "", err
}
var newRecord objectData
newRecord.Address = prm.Address
@ -82,24 +82,24 @@ func (b *BlobTree) saveToFile(prm common.PutPrm, dir string) (uint64, error) {
size, err := b.writeToTmpAndRename(append(currentContent, newRecord), path)
if err != nil {
return 0, err
return "", err
}
returnIdx = size < b.cfg.targetFileSizeBytes
return idx, nil
return path, nil
}
func (b *BlobTree) writeToTmpAndRename(records []objectData, path string) (uint64, error) {
tmpFile := path + tempFileSymbols + strconv.FormatUint(b.suffix.Add(1), 16)
tmpPath := path + tempFileSymbols + strconv.FormatUint(b.suffix.Add(1), 16)
size, err := b.saveContentToFile(records, tmpFile)
size, err := b.saveContentToFile(records, tmpPath)
if err != nil {
_ = os.Remove(tmpFile)
_ = os.Remove(b.getSystemPath(tmpPath))
return 0, err
}
newFile := false
_, err = os.Stat(path)
_, err = os.Stat(b.getSystemPath(path))
if err != nil {
if os.IsNotExist(err) {
newFile = true
@ -108,8 +108,8 @@ func (b *BlobTree) writeToTmpAndRename(records []objectData, path string) (uint6
}
}
if err := os.Rename(tmpFile, path); err != nil {
_ = os.Remove(tmpFile)
if err := os.Rename(b.getSystemPath(tmpPath), b.getSystemPath(path)); err != nil {
_ = os.Remove(b.getSystemPath(tmpPath))
return 0, err
}

25
pkg/local_object_storage/blobstor/blobtree/storage_id.go
View file

@ -1,28 +1,15 @@
package blobtree
import (
"encoding/binary"
"strconv"
"strings"
)
const storageIDLength = 8
const storageIDPrefix = "blobtree:"
func tryParseIdxFromStorageID(storageID []byte) (uint64, bool) {
if len(storageID) == storageIDLength {
return binary.LittleEndian.Uint64(storageID), true
}
return 0, false
func getPathFromStorageID(storageID []byte) (string, bool) {
return strings.CutPrefix(string(storageID), storageIDPrefix)
}
func storageIDToIdxStringSafe(storageID []byte) string {
if len(storageID) == storageIDLength {
return strconv.FormatUint(binary.LittleEndian.Uint64(storageID), 10)
}
return ""
}
func idxToStorageID(idx uint64) []byte {
storageID := make([]byte, storageIDLength)
binary.LittleEndian.PutUint64(storageID, idx)
return storageID
func pathToStorageID(path string) []byte {
return []byte(storageIDPrefix + path)
}