package meta import ( "bytes" "context" "errors" "fmt" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) var errFailedToRemoveUniqueIndexes = errors.New("can't remove unique indexes") // DeletePrm groups the parameters of Delete operation. type DeletePrm struct { addrs []oid.Address } // DeleteRes groups the resulting values of Delete operation. type DeleteRes struct { phyCount uint64 logicCount uint64 userCount uint64 phySize uint64 logicSize uint64 removedByCnrID map[cid.ID]ObjectCounters } // LogicCount returns the number of removed logic // objects. func (d DeleteRes) LogicCount() uint64 { return d.logicCount } func (d DeleteRes) UserCount() uint64 { return d.userCount } // RemovedByCnrID returns the number of removed objects by container ID. func (d DeleteRes) RemovedByCnrID() map[cid.ID]ObjectCounters { return d.removedByCnrID } // PhyCount returns the number of removed physical objects. func (d DeleteRes) PhyCount() uint64 { return d.phyCount } // PhySize returns the size of removed physical objects. func (d DeleteRes) PhySize() uint64 { return d.phySize } // LogicSize returns the size of removed logical objects. func (d DeleteRes) LogicSize() uint64 { return d.logicSize } // SetAddresses is a Delete option to set the addresses of the objects to delete. // // Option is required. func (p *DeletePrm) SetAddresses(addrs ...oid.Address) { p.addrs = addrs } type referenceNumber struct { all, cur int obj *objectSDK.Object } type referenceCounter map[string]*referenceNumber // Delete removed object records from metabase indexes. func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { var ( startedAt = time.Now() deleted = false ) defer func() { db.metrics.AddMethodDuration("Delete", time.Since(startedAt), deleted) }() _, span := tracing.StartSpanFromContext(ctx, "metabase.Delete", trace.WithAttributes( attribute.Int("addr_count", len(prm.addrs)), )) defer span.End() db.modeMtx.RLock() defer db.modeMtx.RUnlock() if db.mode.NoMetabase() { return DeleteRes{}, ErrDegradedMode } else if db.mode.ReadOnly() { return DeleteRes{}, ErrReadOnlyMode } var err error var res DeleteRes err = db.boltDB.Batch(func(tx *bbolt.Tx) error { res, err = db.deleteGroup(tx, prm.addrs) return err }) if err == nil { deleted = true for i := range prm.addrs { storagelog.Write(ctx, db.log, storagelog.AddressField(prm.addrs[i]), storagelog.OpField("metabase DELETE")) } } return res, metaerr.Wrap(err) } // deleteGroup deletes object from the metabase. Handles removal of the // references of the split objects. func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) { res := DeleteRes{ removedByCnrID: make(map[cid.ID]ObjectCounters), } refCounter := make(referenceCounter, len(addrs)) currEpoch := db.epochState.CurrentEpoch() for i := range addrs { r, err := db.delete(tx, addrs[i], refCounter, currEpoch) if err != nil { return DeleteRes{}, err } applyDeleteSingleResult(r, &res, addrs, i) } if err := db.updateCountersDelete(tx, res); err != nil { return DeleteRes{}, err } for _, refNum := range refCounter { if refNum.cur == refNum.all { err := db.deleteObject(tx, refNum.obj, true) if err != nil { return DeleteRes{}, err } } } return res, nil } func (db *DB) updateCountersDelete(tx *bbolt.Tx, res DeleteRes) error { if res.phyCount > 0 { err := db.updateShardObjectCounter(tx, phy, res.phyCount, false) if err != nil { return fmt.Errorf("decrease phy object counter: %w", err) } } if res.logicCount > 0 { err := db.updateShardObjectCounter(tx, logical, res.logicCount, false) if err != nil { return fmt.Errorf("decrease logical object counter: %w", err) } } if res.userCount > 0 { err := db.updateShardObjectCounter(tx, user, res.userCount, false) if err != nil { return fmt.Errorf("decrease user object counter: %w", err) } } if err := db.updateContainerCounter(tx, res.removedByCnrID, false); err != nil { return fmt.Errorf("decrease container object counter: %w", err) } return nil } func applyDeleteSingleResult(r deleteSingleResult, res *DeleteRes, addrs []oid.Address, i int) { if r.Phy { if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { v.Phy++ res.removedByCnrID[addrs[i].Container()] = v } else { res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ Phy: 1, } } res.phyCount++ res.phySize += r.Size } if r.Logic { if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { v.Logic++ res.removedByCnrID[addrs[i].Container()] = v } else { res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ Logic: 1, } } res.logicCount++ res.logicSize += r.Size } if r.User { if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { v.User++ res.removedByCnrID[addrs[i].Container()] = v } else { res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ User: 1, } } res.userCount++ } } type deleteSingleResult struct { Phy bool Logic bool User bool Size uint64 } // delete removes object indexes from the metabase. Counts the references // of the object that is being removed. // The first return value indicates if an object has been removed. (removing a // non-exist object is error-free). The second return value indicates if an // object was available before the removal (for calculating the logical object // counter). The third return value The fourth return value is removed object payload size. func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter, currEpoch uint64) (deleteSingleResult, error) { key := make([]byte, addressKeySize) addrKey := addressKey(addr, key) garbageBKT := tx.Bucket(garbageBucketName) graveyardBKT := tx.Bucket(graveyardBucketName) removeAvailableObject := inGraveyardWithKey(addrKey, graveyardBKT, garbageBKT) == 0 // unmarshal object, work only with physically stored (raw == true) objects obj, err := db.get(tx, addr, key, false, true, currEpoch) if err != nil { if client.IsErrObjectNotFound(err) { addrKey = addressKey(addr, key) if garbageBKT != nil { err := garbageBKT.Delete(addrKey) if err != nil { return deleteSingleResult{}, fmt.Errorf("remove from garbage bucket: %w", err) } } return deleteSingleResult{}, nil } var siErr *objectSDK.SplitInfoError var ecErr *objectSDK.ECInfoError if errors.As(err, &siErr) || errors.As(err, &ecErr) { // if object is virtual (parent) then do nothing, it will be deleted with last child // if object is erasure-coded it will be deleted with the last chunk presented on the shard return deleteSingleResult{}, nil } return deleteSingleResult{}, err } addrKey = addressKey(addr, key) // remove record from the garbage bucket if garbageBKT != nil { err := garbageBKT.Delete(addrKey) if err != nil { return deleteSingleResult{}, fmt.Errorf("remove from garbage bucket: %w", err) } } // if object is an only link to a parent, then remove parent if parent := obj.Parent(); parent != nil { parAddr := object.AddressOf(parent) sParAddr := addressKey(parAddr, key) k := string(sParAddr) nRef, ok := refCounter[k] if !ok { nRef = &referenceNumber{ all: parentLength(tx, parAddr), obj: parent, } refCounter[k] = nRef } nRef.cur++ } isUserObject := IsUserObject(obj) // remove object err = db.deleteObject(tx, obj, false) if err != nil { return deleteSingleResult{}, fmt.Errorf("remove object: %w", err) } if err := deleteECRelatedInfo(tx, garbageBKT, obj, addr.Container(), refCounter); err != nil { return deleteSingleResult{}, err } return deleteSingleResult{ Phy: true, Logic: removeAvailableObject, User: isUserObject && removeAvailableObject, Size: obj.PayloadSize(), }, nil } func (db *DB) deleteObject( tx *bbolt.Tx, obj *objectSDK.Object, isParent bool, ) error { err := delUniqueIndexes(tx, obj, isParent) if err != nil { return errFailedToRemoveUniqueIndexes } err = updateListIndexes(tx, obj, delListIndexItem) if err != nil { return fmt.Errorf("remove list indexes: %w", err) } err = updateFKBTIndexes(tx, obj, delFKBTIndexItem) if err != nil { return fmt.Errorf("remove fake bucket tree indexes: %w", err) } if isParent { // remove record from the garbage bucket, because regular object deletion does nothing for virtual object garbageBKT := tx.Bucket(garbageBucketName) if garbageBKT != nil { key := make([]byte, addressKeySize) addrKey := addressKey(object.AddressOf(obj), key) err := garbageBKT.Delete(addrKey) if err != nil { return fmt.Errorf("remove from garbage bucket: %w", err) } } } return nil } // parentLength returns amount of available children from parentid index. func parentLength(tx *bbolt.Tx, addr oid.Address) int { bucketName := make([]byte, bucketKeySize) bkt := tx.Bucket(parentBucketName(addr.Container(), bucketName[:])) if bkt == nil { return 0 } lst, err := decodeList(bkt.Get(objectKey(addr.Object(), bucketName[:]))) if err != nil { return 0 } return len(lst) } func delUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) { bkt := tx.Bucket(item.name) if bkt != nil { _ = bkt.Delete(item.key) // ignore error, best effort there } } func delListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { bkt := tx.Bucket(item.name) if bkt == nil { return nil } lst, err := decodeList(bkt.Get(item.key)) if err != nil || len(lst) == 0 { return nil } // remove element from the list for i := range lst { if bytes.Equal(item.val, lst[i]) { copy(lst[i:], lst[i+1:]) lst = lst[:len(lst)-1] break } } // if list empty, remove the key from bucket if len(lst) == 0 { _ = bkt.Delete(item.key) // ignore error, best effort there return nil } // if list is not empty, then update it encodedLst, err := encodeList(lst) if err != nil { return nil // ignore error, best effort there } _ = bkt.Put(item.key, encodedLst) // ignore error, best effort there return nil } func delFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { bkt := tx.Bucket(item.name) if bkt == nil { return nil } fkbtRoot := bkt.Bucket(item.key) if fkbtRoot == nil { return nil } if err := fkbtRoot.Delete(item.val); err != nil { return err } if hasAnyItem(fkbtRoot) { return nil } if err := bkt.DeleteBucket(item.key); err != nil { return err } if hasAnyItem(bkt) { return nil } return tx.DeleteBucket(item.name) } func hasAnyItem(b *bbolt.Bucket) bool { var hasAnyItem bool c := b.Cursor() for k, _ := c.First(); k != nil; { hasAnyItem = true break } return hasAnyItem } func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error { addr := object.AddressOf(obj) objKey := objectKey(addr.Object(), make([]byte, objectKeySize)) cnr := addr.Container() bucketName := make([]byte, bucketKeySize) // add value to primary unique bucket if !isParent { switch obj.Type() { case objectSDK.TypeRegular: bucketName = primaryBucketName(cnr, bucketName) case objectSDK.TypeTombstone: bucketName = tombstoneBucketName(cnr, bucketName) case objectSDK.TypeLock: bucketName = bucketNameLockers(cnr, bucketName) default: return ErrUnknownObjectType } delUniqueIndexItem(tx, namedBucketItem{ name: bucketName, key: objKey, }) } else { delUniqueIndexItem(tx, namedBucketItem{ name: parentBucketName(cnr, bucketName), key: objKey, }) } delUniqueIndexItem(tx, namedBucketItem{ // remove from storage id index name: smallBucketName(cnr, bucketName), key: objKey, }) delUniqueIndexItem(tx, namedBucketItem{ // remove from root index name: rootBucketName(cnr, bucketName), key: objKey, }) if expEpoch, ok := hasExpirationEpoch(obj); ok { delUniqueIndexItem(tx, namedBucketItem{ name: expEpochToObjectBucketName, key: expirationEpochKey(expEpoch, cnr, addr.Object()), }) delUniqueIndexItem(tx, namedBucketItem{ name: objectToExpirationEpochBucketName(cnr, make([]byte, bucketKeySize)), key: objKey, }) } return nil } func deleteECRelatedInfo(tx *bbolt.Tx, garbageBKT *bbolt.Bucket, obj *objectSDK.Object, cnr cid.ID, refCounter referenceCounter) error { ech := obj.ECHeader() if ech == nil { return nil } hasAnyChunks := hasAnyECChunks(tx, ech, cnr) // drop EC parent GC mark if current EC chunk is the last one if !hasAnyChunks && garbageBKT != nil { var ecParentAddress oid.Address ecParentAddress.SetContainer(cnr) ecParentAddress.SetObject(ech.Parent()) addrKey := addressKey(ecParentAddress, make([]byte, addressKeySize)) err := garbageBKT.Delete(addrKey) if err != nil { return fmt.Errorf("remove EC parent from garbage bucket: %w", err) } } // also drop EC parent root info if current EC chunk is the last one if !hasAnyChunks { delUniqueIndexItem(tx, namedBucketItem{ name: rootBucketName(cnr, make([]byte, bucketKeySize)), key: objectKey(ech.Parent(), make([]byte, objectKeySize)), }) } if ech.ParentSplitParentID() == nil { return nil } var splitParentAddress oid.Address splitParentAddress.SetContainer(cnr) splitParentAddress.SetObject(*ech.ParentSplitParentID()) if ref, ok := refCounter[string(addressKey(splitParentAddress, make([]byte, addressKeySize)))]; ok { // linking object is already processing // so just inform that one more reference was deleted // split info and gc marks will be deleted after linking object delete ref.cur++ return nil } if parentLength(tx, splitParentAddress) > 0 { // linking object still exists, so leave split info and gc mark deletion for linking object processing return nil } // drop split parent gc mark if garbageBKT != nil { addrKey := addressKey(splitParentAddress, make([]byte, addressKeySize)) err := garbageBKT.Delete(addrKey) if err != nil { return fmt.Errorf("remove EC parent from garbage bucket: %w", err) } } // drop split info delUniqueIndexItem(tx, namedBucketItem{ name: rootBucketName(cnr, make([]byte, bucketKeySize)), key: objectKey(*ech.ParentSplitParentID(), make([]byte, objectKeySize)), }) return nil } func hasAnyECChunks(tx *bbolt.Tx, ech *objectSDK.ECHeader, cnr cid.ID) bool { data := getFromBucket(tx, ecInfoBucketName(cnr, make([]byte, bucketKeySize)), objectKey(ech.Parent(), make([]byte, objectKeySize))) return len(data) > 0 }