realloc/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#653] Support removal old combined objects

Browse source 
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
Denis Kirillov 2025-03-26 09:55:46 +03:00
parent 4a430257a4
commit 396c89f7bc
8 changed files with 88 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

22
api/layer/tombstone.go
View file

@ -24,6 +24,10 @@ import (
)
func (n *Layer) putTombstones(ctx context.Context, bkt *data.BucketInfo, networkInfo netmap.NetworkInfo, members []oid.ID) error {
return n.putTombstonesBase(ctx, bkt, networkInfo, members, frostfs.PrmAuth{})
}
func (n *Layer) putTombstonesBase(ctx context.Context, bkt *data.BucketInfo, networkInfo netmap.NetworkInfo, members []oid.ID, prmAuth frostfs.PrmAuth) error {
if len(members) == 0 {
return nil
}
@ -42,7 +46,7 @@ func (n *Layer) putTombstones(ctx context.Context, bkt *data.BucketInfo, network
if end > len(members) {
end = len(members)
}
n.submitPutTombstone(ctx, bkt, members[tombstoneMembersSize*i:end], networkInfo.CurrentEpoch()+tombstoneLifetime, &wg, errCh)
n.submitPutTombstone(ctx, bkt, members[tombstoneMembersSize*i:end], networkInfo.CurrentEpoch()+tombstoneLifetime, &wg, prmAuth, errCh)
}
wg.Wait()
@ -55,7 +59,7 @@ func (n *Layer) putTombstones(ctx context.Context, bkt *data.BucketInfo, network
return nil
}
func (n *Layer) submitPutTombstone(ctx context.Context, bkt *data.BucketInfo, members []oid.ID, expEpoch uint64, wg *sync.WaitGroup, errCh chan<- error) {
func (n *Layer) submitPutTombstone(ctx context.Context, bkt *data.BucketInfo, members []oid.ID, expEpoch uint64, wg *sync.WaitGroup, prmAuth frostfs.PrmAuth, errCh chan<- error) {
tomb := object.NewTombstone()
tomb.SetExpirationEpoch(expEpoch)
tomb.SetMembers(members)
@ -64,7 +68,7 @@ func (n *Layer) submitPutTombstone(ctx context.Context, bkt *data.BucketInfo, me
err := n.workerPool.Submit(func() {
defer wg.Done()
if err := n.putTombstoneObject(ctx, tomb, bkt); err != nil {
if err := n.putTombstoneObject(ctx, tomb, bkt, prmAuth); err != nil {
n.reqLogger(ctx).Warn(logs.FailedToPutTombstoneObject, zap.String("cid", bkt.CID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
errCh <- fmt.Errorf("put tombstone object: %w", err)
}
@ -76,7 +80,7 @@ func (n *Layer) submitPutTombstone(ctx context.Context, bkt *data.BucketInfo, me
}
}
func (n *Layer) putTombstoneObject(ctx context.Context, tomb *object.Tombstone, bktInfo *data.BucketInfo) error {
func (n *Layer) putTombstoneObject(ctx context.Context, tomb *object.Tombstone, bktInfo *data.BucketInfo, prmAuth frostfs.PrmAuth) error {
payload, err := tomb.Marshal()
if err != nil {
return fmt.Errorf("marshal tombstone: %w", err)
@ -91,6 +95,7 @@ func (n *Layer) putTombstoneObject(ctx context.Context, tomb *object.Tombstone,
WithoutHomomorphicHash: bktInfo.HomomorphicHashDisabled,
BufferMaxSize: n.features.BufferMaxSizeForPut(),
Type: object.TypeTombstone,
PrmAuth: prmAuth,
}
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
@ -113,8 +118,17 @@ func (n *Layer) getMembers(ctx context.Context, cnrID cid.ID, objID oid.ID, toke
}
func prepareTokensParameter(ctx context.Context, bktOwner user.ID) relations.Tokens {
return prepareTokensParameterBase(ctx, bktOwner, frostfs.PrmAuth{})
}
func prepareTokensParameterBase(ctx context.Context, bktOwner user.ID, prmAuth frostfs.PrmAuth) relations.Tokens {
tokens := relations.Tokens{}
if prmAuth.BearerToken != nil {
tokens.Bearer = prmAuth.BearerToken
return tokens
}
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate.BearerToken != nil {
if bd.Gate.BearerToken.Impersonate() || bktOwner.Equals(bearer.ResolveIssuer(*bd.Gate.BearerToken)) {
tokens.Bearer = bd.Gate.BearerToken