diff --git a/pkg/local_object_storage/blobstor/blobtree/blobtree.go b/pkg/local_object_storage/blobstor/blobtree/blobtree.go index 1270c54bf..a50114676 100644 --- a/pkg/local_object_storage/blobstor/blobtree/blobtree.go +++ b/pkg/local_object_storage/blobstor/blobtree/blobtree.go @@ -48,27 +48,29 @@ func New(opts ...Option) *BlobTree { return b } -func (b *BlobTree) getDirectoryPath(addr oid.Address) string { +func (b *BlobTree) getDir(addr oid.Address) string { sAddr := addr.Object().EncodeToString() + "." + addr.Container().EncodeToString() var sb strings.Builder - size := int(1+b.cfg.depth*(directoryLength+1)) + len(b.cfg.rootPath) // /path + slash + (character + slash for every level) + size := int(b.cfg.depth * (directoryLength + 1)) // (character + slash for every level) sb.Grow(size) - sb.WriteString(b.cfg.rootPath) for i := uint64(0); i < b.cfg.depth; i++ { - sb.WriteRune(filepath.Separator) + if i > 0 { + sb.WriteRune(filepath.Separator) + } sb.WriteString(sAddr[:directoryLength]) sAddr = sAddr[directoryLength:] } - - sb.WriteRune(filepath.Separator) return sb.String() } -func (b *BlobTree) createDir(dir string) error { +func (b *BlobTree) createDir(dir string, isSystemPath bool) error { b.dirLock.Lock(dir) defer b.dirLock.Unlock(dir) + if !isSystemPath { + dir = b.getSystemPath(dir) + } if err := util.MkdirAllX(dir, b.cfg.permissions); err != nil { if errors.Is(err, syscall.ENOSPC) { err = common.ErrNoSpace diff --git a/pkg/local_object_storage/blobstor/blobtree/content.go b/pkg/local_object_storage/blobstor/blobtree/content.go index c59ff4ed6..b3a457e54 100644 --- a/pkg/local_object_storage/blobstor/blobtree/content.go +++ b/pkg/local_object_storage/blobstor/blobtree/content.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strconv" + "strings" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -41,7 +42,7 @@ type objectData struct { } func (b *BlobTree) readFileContent(path string) ([]objectData, error) { - rawData, err := os.ReadFile(path) + rawData, err := os.ReadFile(b.getSystemPath(path)) if err != nil { if os.IsNotExist(err) { return []objectData{}, nil @@ -130,6 +131,7 @@ func (b *BlobTree) saveContentToFile(records []objectData, path string) (uint64, } func (b *BlobTree) writeFile(p string, data []byte) error { + p = b.getSystemPath(p) f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_SYNC, b.cfg.permissions) if err != nil { return err @@ -141,6 +143,10 @@ func (b *BlobTree) writeFile(p string, data []byte) error { return err } +func (b *BlobTree) getSystemPath(path string) string { + return filepath.Join(b.cfg.rootPath, path) +} + func (b *BlobTree) marshalSlice(records []objectData) ([]byte, error) { buf := make([]byte, b.estimateSize(records)) result := buf @@ -191,3 +197,13 @@ func (b *BlobTree) estimateSize(records []objectData) uint64 { func (b *BlobTree) getFilePath(dir string, idx uint64) string { return filepath.Join(dir, strconv.FormatUint(idx, 16)+dataExtension) } + +func (b *BlobTree) parsePath(path string) (string, uint64, error) { + dir := filepath.Dir(path) + fileName := strings.TrimSuffix(filepath.Base(path), dataExtension) + idx, err := strconv.ParseUint(fileName, 16, 64) + if err != nil { + return "", 0, fmt.Errorf("failed to parse blobtree path: %w", err) + } + return dir, idx, nil +} diff --git a/pkg/local_object_storage/blobstor/blobtree/control.go b/pkg/local_object_storage/blobstor/blobtree/control.go index d2f8f3fc2..584421b43 100644 --- a/pkg/local_object_storage/blobstor/blobtree/control.go +++ b/pkg/local_object_storage/blobstor/blobtree/control.go @@ -19,37 +19,35 @@ func (b *BlobTree) Open(readOnly bool) error { } func (b *BlobTree) Init() error { - if err := b.createDir(b.cfg.rootPath); err != nil { + if err := b.createDir(b.cfg.rootPath, true); err != nil { return err } var eg errgroup.Group eg.SetLimit(b.cfg.initWorkersCount) eg.Go(func() error { - return b.initDir(&eg, b.cfg.rootPath, 0) + return b.initDir(&eg, "") }) return eg.Wait() } -func (b *BlobTree) initDir(eg *errgroup.Group, dir string, depth uint64) error { - entities, err := os.ReadDir(dir) +func (b *BlobTree) initDir(eg *errgroup.Group, dir string) error { + entities, err := os.ReadDir(b.getSystemPath(dir)) if err != nil { return err } for _, entity := range entities { - if depth < b.cfg.depth && entity.IsDir() { + if entity.IsDir() { eg.Go(func() error { - return b.initDir(eg, filepath.Join(dir, entity.Name()), depth+1) + return b.initDir(eg, filepath.Join(dir, entity.Name())) }) continue } - if depth != b.cfg.depth { - continue - } + path := filepath.Join(dir, entity.Name()) if b.isTempFile(entity.Name()) { - if err = os.Remove(filepath.Join(dir, entity.Name())); err != nil { + if err = os.Remove(b.getSystemPath(path)); err != nil { return err } continue @@ -57,12 +55,12 @@ func (b *BlobTree) initDir(eg *errgroup.Group, dir string, depth uint64) error { idx, err := b.parseIdx(entity.Name()) if err != nil { - continue + return err } b.dispatcher.Init(dir, idx) b.cfg.metrics.IncFilesCount() - stat, err := os.Stat(filepath.Join(dir, entity.Name())) + stat, err := os.Stat(b.getSystemPath(path)) if err != nil { return err } diff --git a/pkg/local_object_storage/blobstor/blobtree/delete.go b/pkg/local_object_storage/blobstor/blobtree/delete.go index cdda9d39f..bfc46ef2b 100644 --- a/pkg/local_object_storage/blobstor/blobtree/delete.go +++ b/pkg/local_object_storage/blobstor/blobtree/delete.go @@ -3,6 +3,7 @@ package blobtree import ( "context" "os" + "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -27,7 +28,7 @@ func (b *BlobTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Del trace.WithAttributes( attribute.String("path", b.cfg.rootPath), attribute.String("address", prm.Address.EncodeToString()), - attribute.String("storage_id", storageIDToIdxStringSafe(prm.StorageID)), + attribute.String("storage_id", string(prm.StorageID)), )) defer span.End() @@ -37,8 +38,8 @@ func (b *BlobTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Del var res common.DeleteRes var err error - if idx, ok := tryParseIdxFromStorageID(prm.StorageID); ok { - res, err = b.deleteFromIdx(prm.Address, idx) + if path, ok := getPathFromStorageID(prm.StorageID); ok { + res, err = b.deleteFromPath(prm.Address, path) } else { res, err = b.findAndDelete(prm.Address) } @@ -46,13 +47,15 @@ func (b *BlobTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Del return res, err } -func (b *BlobTree) deleteFromIdx(addr oid.Address, idx uint64) (common.DeleteRes, error) { - dir := b.getDirectoryPath(addr) - path := b.getFilePath(dir, idx) - +func (b *BlobTree) deleteFromPath(addr oid.Address, path string) (common.DeleteRes, error) { b.fileLock.Lock(path) defer b.fileLock.Unlock(path) + dir, idx, err := b.parsePath(path) + if err != nil { + return common.DeleteRes{}, err + } + records, err := b.readFileContent(path) if err != nil { return common.DeleteRes{}, err @@ -71,7 +74,7 @@ func (b *BlobTree) deleteFromIdx(addr oid.Address, idx uint64) (common.DeleteRes } if len(records) == 1 { - err = os.Remove(path) + err = os.Remove(b.getSystemPath(path)) if err == nil { b.dispatcher.ReturnIdx(dir, idx) b.cfg.metrics.DecFilesCount() @@ -79,7 +82,7 @@ func (b *BlobTree) deleteFromIdx(addr oid.Address, idx uint64) (common.DeleteRes return common.DeleteRes{}, err } - records = append(records[:idx], records[idx+1:]...) + records = append(records[:deleteIdx], records[deleteIdx+1:]...) size, err := b.writeToTmpAndRename(records, path) if err != nil { return common.DeleteRes{}, err @@ -91,16 +94,16 @@ func (b *BlobTree) deleteFromIdx(addr oid.Address, idx uint64) (common.DeleteRes } func (b *BlobTree) findAndDelete(addr oid.Address) (common.DeleteRes, error) { - dir := b.getDirectoryPath(addr) + dir := b.getDir(addr) idx, err := b.findFileIdx(dir, addr) if err != nil { return common.DeleteRes{}, err } - return b.deleteFromIdx(addr, idx) + return b.deleteFromPath(addr, b.getFilePath(dir, idx)) } func (b *BlobTree) findFileIdx(dir string, addr oid.Address) (uint64, error) { - entities, err := os.ReadDir(dir) + entities, err := os.ReadDir(filepath.Join(b.cfg.rootPath, dir)) if err != nil { if os.IsNotExist(err) { return 0, logicerr.Wrap(new(apistatus.ObjectNotFound)) diff --git a/pkg/local_object_storage/blobstor/blobtree/exists.go b/pkg/local_object_storage/blobstor/blobtree/exists.go index 433578b32..559cbf57e 100644 --- a/pkg/local_object_storage/blobstor/blobtree/exists.go +++ b/pkg/local_object_storage/blobstor/blobtree/exists.go @@ -26,25 +26,22 @@ func (b *BlobTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi trace.WithAttributes( attribute.String("path", b.cfg.rootPath), attribute.String("address", prm.Address.EncodeToString()), - attribute.String("storage_id", storageIDToIdxStringSafe(prm.StorageID)), + attribute.String("storage_id", string(prm.StorageID)), )) defer span.End() var res common.ExistsRes var err error - if idx, ok := tryParseIdxFromStorageID(prm.StorageID); ok { - res, err = b.existsFromIdx(prm.Address, idx) + if path, ok := getPathFromStorageID(prm.StorageID); ok { + res, err = b.existsFromPath(prm.Address, path) } else { - res, err = b.findAndCheck(prm.Address) + res, err = b.findAndCheckExistance(prm.Address) } success = err == nil return res, err } -func (b *BlobTree) existsFromIdx(addr oid.Address, idx uint64) (common.ExistsRes, error) { - dir := b.getDirectoryPath(addr) - path := b.getFilePath(dir, idx) - +func (b *BlobTree) existsFromPath(addr oid.Address, path string) (common.ExistsRes, error) { b.fileLock.RLock(path) defer b.fileLock.RUnlock(path) @@ -64,15 +61,16 @@ func (b *BlobTree) existsFromIdx(addr oid.Address, idx uint64) (common.ExistsRes return common.ExistsRes{}, nil } -func (b *BlobTree) findAndCheck(addr oid.Address) (common.ExistsRes, error) { - dir := b.getDirectoryPath(addr) +func (b *BlobTree) findAndCheckExistance(addr oid.Address) (common.ExistsRes, error) { + dir := b.getDir(addr) _, err := b.findFileIdx(dir, addr) - if err != nil { - var notFound *apistatus.ObjectNotFound - if errors.As(err, ¬Found) { - return common.ExistsRes{}, nil - } - return common.ExistsRes{}, err + if err == nil { + return common.ExistsRes{Exists: true}, nil } - return common.ExistsRes{Exists: true}, nil + + var notFound *apistatus.ObjectNotFound + if errors.As(err, ¬Found) { + return common.ExistsRes{}, nil + } + return common.ExistsRes{}, err } diff --git a/pkg/local_object_storage/blobstor/blobtree/get.go b/pkg/local_object_storage/blobstor/blobtree/get.go index 1569a0687..de92b0239 100644 --- a/pkg/local_object_storage/blobstor/blobtree/get.go +++ b/pkg/local_object_storage/blobstor/blobtree/get.go @@ -3,6 +3,7 @@ package blobtree import ( "context" "os" + "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -29,7 +30,7 @@ func (b *BlobTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, e trace.WithAttributes( attribute.String("path", b.cfg.rootPath), attribute.String("address", prm.Address.EncodeToString()), - attribute.String("storage_id", storageIDToIdxStringSafe(prm.StorageID)), + attribute.String("storage_id", string(prm.StorageID)), attribute.Bool("raw", prm.Raw), )) defer span.End() @@ -41,16 +42,13 @@ func (b *BlobTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, e } func (b *BlobTree) get(prm common.GetPrm) (common.GetRes, error) { - if idx, ok := tryParseIdxFromStorageID(prm.StorageID); ok { - return b.getFromIdx(prm.Address, idx) + if path, ok := getPathFromStorageID(prm.StorageID); ok { + return b.getFromPath(prm.Address, path) } return b.findAndGet(prm.Address) } -func (b *BlobTree) getFromIdx(addr oid.Address, idx uint64) (common.GetRes, error) { - dir := b.getDirectoryPath(addr) - path := b.getFilePath(dir, idx) - +func (b *BlobTree) getFromPath(addr oid.Address, path string) (common.GetRes, error) { b.fileLock.RLock(path) defer b.fileLock.RUnlock(path) @@ -82,8 +80,8 @@ func (b *BlobTree) unmarshalGetRes(record objectData) (common.GetRes, error) { } func (b *BlobTree) findAndGet(addr oid.Address) (common.GetRes, error) { - dir := b.getDirectoryPath(addr) - entities, err := os.ReadDir(dir) + dir := b.getDir(addr) + entities, err := os.ReadDir(filepath.Join(b.cfg.rootPath, dir)) if err != nil { if os.IsNotExist(err) { return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) @@ -97,11 +95,7 @@ func (b *BlobTree) findAndGet(addr oid.Address) (common.GetRes, error) { if b.isTempFile(entity.Name()) { continue } - idx, err := b.parseIdx(entity.Name()) - if err != nil { - continue - } - path := b.getFilePath(dir, idx) + path := filepath.Join(dir, entity.Name()) res, err := b.tryReadObject(path, addr) if err != nil { return common.GetRes{}, err diff --git a/pkg/local_object_storage/blobstor/blobtree/get_range.go b/pkg/local_object_storage/blobstor/blobtree/get_range.go index 0c6071c59..a89693c9f 100644 --- a/pkg/local_object_storage/blobstor/blobtree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobtree/get_range.go @@ -27,7 +27,7 @@ func (b *BlobTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common trace.WithAttributes( attribute.String("path", b.cfg.rootPath), attribute.String("address", prm.Address.EncodeToString()), - attribute.String("storage_id", storageIDToIdxStringSafe(prm.StorageID)), + attribute.String("storage_id", string(prm.StorageID)), attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), attribute.String("length", strconv.FormatUint(prm.Range.GetLength(), 10)), )) diff --git a/pkg/local_object_storage/blobstor/blobtree/iterate.go b/pkg/local_object_storage/blobstor/blobtree/iterate.go index 1fc1418d9..61a00f7d5 100644 --- a/pkg/local_object_storage/blobstor/blobtree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobtree/iterate.go @@ -28,12 +28,12 @@ func (b *BlobTree) Iterate(ctx context.Context, prm common.IteratePrm) (common.I )) defer span.End() - err = b.iterateDir(b.cfg.rootPath, 0, prm) + err = b.iterateDir("", prm) return common.IterateRes{}, err } -func (b *BlobTree) iterateDir(dir string, depth uint64, prm common.IteratePrm) error { - entities, err := os.ReadDir(dir) +func (b *BlobTree) iterateDir(dir string, prm common.IteratePrm) error { + entities, err := os.ReadDir(filepath.Join(b.cfg.rootPath, dir)) if err != nil { if prm.IgnoreErrors { return nil @@ -42,24 +42,20 @@ func (b *BlobTree) iterateDir(dir string, depth uint64, prm common.IteratePrm) e } for _, entity := range entities { - if depth < b.cfg.depth && entity.IsDir() { - err := b.iterateDir(filepath.Join(dir, entity.Name()), depth+1, prm) + if entity.IsDir() { + err := b.iterateDir(filepath.Join(dir, entity.Name()), prm) if err != nil { return err } - } - if depth != b.cfg.depth { continue } + if b.isTempFile(entity.Name()) { continue } - idx, err := b.parseIdx(entity.Name()) - if err != nil { - continue - } - path := b.getFilePath(dir, idx) - err = b.iterateRecords(idx, path, prm) + + path := filepath.Join(dir, entity.Name()) + err = b.iterateRecords(path, prm) if err != nil { return err } @@ -67,7 +63,7 @@ func (b *BlobTree) iterateDir(dir string, depth uint64, prm common.IteratePrm) e return nil } -func (b *BlobTree) iterateRecords(idx uint64, path string, prm common.IteratePrm) error { +func (b *BlobTree) iterateRecords(path string, prm common.IteratePrm) error { b.fileLock.RLock(path) defer b.fileLock.RUnlock(path) @@ -103,7 +99,7 @@ func (b *BlobTree) iterateRecords(idx uint64, path string, prm common.IteratePrm err = prm.Handler(common.IterationElement{ Address: record.Address, ObjectData: record.Data, - StorageID: idxToStorageID(idx), + StorageID: pathToStorageID(path), }) if err != nil { return err diff --git a/pkg/local_object_storage/blobstor/blobtree/put.go b/pkg/local_object_storage/blobstor/blobtree/put.go index 8359b771a..6d6653171 100644 --- a/pkg/local_object_storage/blobstor/blobtree/put.go +++ b/pkg/local_object_storage/blobstor/blobtree/put.go @@ -37,9 +37,9 @@ func (b *BlobTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e return common.PutRes{}, common.ErrReadOnly } - dir := b.getDirectoryPath(prm.Address) + dir := b.getDir(prm.Address) - if err := b.createDir(dir); err != nil { + if err := b.createDir(dir, false); err != nil { return common.PutRes{}, err } @@ -47,7 +47,7 @@ func (b *BlobTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e prm.RawData = b.compressor.Compress(prm.RawData) } - idx, err := b.saveToFile(prm, dir) + path, err := b.saveToLocalDir(prm, dir) if err != nil { return common.PutRes{}, err } @@ -55,10 +55,10 @@ func (b *BlobTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e success = true size = len(prm.RawData) - return common.PutRes{StorageID: idxToStorageID(idx)}, nil + return common.PutRes{StorageID: pathToStorageID(path)}, nil } -func (b *BlobTree) saveToFile(prm common.PutPrm, dir string) (uint64, error) { +func (b *BlobTree) saveToLocalDir(prm common.PutPrm, dir string) (string, error) { returnIdx := true idx := b.dispatcher.GetIdxForWrite(dir) path := b.getFilePath(dir, idx) @@ -74,7 +74,7 @@ func (b *BlobTree) saveToFile(prm common.PutPrm, dir string) (uint64, error) { currentContent, err := b.readFileContent(path) if err != nil { - return 0, err + return "", err } var newRecord objectData newRecord.Address = prm.Address @@ -82,24 +82,24 @@ func (b *BlobTree) saveToFile(prm common.PutPrm, dir string) (uint64, error) { size, err := b.writeToTmpAndRename(append(currentContent, newRecord), path) if err != nil { - return 0, err + return "", err } returnIdx = size < b.cfg.targetFileSizeBytes - return idx, nil + return path, nil } func (b *BlobTree) writeToTmpAndRename(records []objectData, path string) (uint64, error) { - tmpFile := path + tempFileSymbols + strconv.FormatUint(b.suffix.Add(1), 16) + tmpPath := path + tempFileSymbols + strconv.FormatUint(b.suffix.Add(1), 16) - size, err := b.saveContentToFile(records, tmpFile) + size, err := b.saveContentToFile(records, tmpPath) if err != nil { - _ = os.Remove(tmpFile) + _ = os.Remove(b.getSystemPath(tmpPath)) return 0, err } newFile := false - _, err = os.Stat(path) + _, err = os.Stat(b.getSystemPath(path)) if err != nil { if os.IsNotExist(err) { newFile = true @@ -108,8 +108,8 @@ func (b *BlobTree) writeToTmpAndRename(records []objectData, path string) (uint6 } } - if err := os.Rename(tmpFile, path); err != nil { - _ = os.Remove(tmpFile) + if err := os.Rename(b.getSystemPath(tmpPath), b.getSystemPath(path)); err != nil { + _ = os.Remove(b.getSystemPath(tmpPath)) return 0, err } diff --git a/pkg/local_object_storage/blobstor/blobtree/storage_id.go b/pkg/local_object_storage/blobstor/blobtree/storage_id.go index cd176980b..64a268eb2 100644 --- a/pkg/local_object_storage/blobstor/blobtree/storage_id.go +++ b/pkg/local_object_storage/blobstor/blobtree/storage_id.go @@ -1,28 +1,15 @@ package blobtree import ( - "encoding/binary" - "strconv" + "strings" ) -const storageIDLength = 8 +const storageIDPrefix = "blobtree:" -func tryParseIdxFromStorageID(storageID []byte) (uint64, bool) { - if len(storageID) == storageIDLength { - return binary.LittleEndian.Uint64(storageID), true - } - return 0, false +func getPathFromStorageID(storageID []byte) (string, bool) { + return strings.CutPrefix(string(storageID), storageIDPrefix) } -func storageIDToIdxStringSafe(storageID []byte) string { - if len(storageID) == storageIDLength { - return strconv.FormatUint(binary.LittleEndian.Uint64(storageID), 10) - } - return "" -} - -func idxToStorageID(idx uint64) []byte { - storageID := make([]byte, storageIDLength) - binary.LittleEndian.PutUint64(storageID, idx) - return storageID +func pathToStorageID(path string) []byte { + return []byte(storageIDPrefix + path) }