From 36efccd86251a8445e6efe1550d342db10b0230a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Aug 2024 15:45:53 +0300 Subject: [PATCH] [#1298] writecache: Add `shrink` flag for Seal command Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/writecache.go | 8 ++- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/writecache.go | 3 +- pkg/local_object_storage/shard/writecache.go | 3 +- pkg/local_object_storage/writecache/flush.go | 2 +- pkg/local_object_storage/writecache/mode.go | 55 +++++++++++++++--- pkg/local_object_storage/writecache/seal.go | 4 +- .../writecache/writecache.go | 1 + .../control/server/seal_writecache.go | 1 + pkg/services/control/service.pb.go | Bin 270827 -> 271229 bytes pkg/services/control/service.proto | 3 + pkg/services/control/service_frostfs.pb.go | Bin 124152 -> 124245 bytes 12 files changed, 68 insertions(+), 13 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/writecache.go b/cmd/frostfs-cli/modules/control/writecache.go index a665ccae8..b725d8471 100644 --- a/cmd/frostfs-cli/modules/control/writecache.go +++ b/cmd/frostfs-cli/modules/control/writecache.go @@ -9,7 +9,10 @@ import ( "github.com/spf13/cobra" ) -const restoreModeFlag = "restore-mode" +const ( + restoreModeFlag = "restore-mode" + shrinkFlag = "shrink" +) var writecacheShardCmd = &cobra.Command{ Use: "writecache", @@ -29,11 +32,13 @@ func sealWritecache(cmd *cobra.Command, _ []string) { ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) restoreMode, _ := cmd.Flags().GetBool(restoreModeFlag) + shrink, _ := cmd.Flags().GetBool(shrinkFlag) req := &control.SealWriteCacheRequest{Body: &control.SealWriteCacheRequest_Body{ Shard_ID: getShardIDList(cmd), IgnoreErrors: ignoreErrors, RestoreMode: restoreMode, + Shrink: shrink, }} signRequest(cmd, pk, req) @@ -73,6 +78,7 @@ func initControlShardsWritecacheCmd() { ff.Bool(shardAllFlag, false, "Process all shards") ff.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects") ff.Bool(restoreModeFlag, false, "Restore writecache's mode after sealing") + ff.Bool(shrinkFlag, false, "Shrink writecache's internal storage") sealWritecacheShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 67f173f29..ebb822e1c 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -539,4 +539,5 @@ const ( PolicerCouldNotGetChunk = "could not get EC chunk" PolicerCouldNotGetChunks = "could not get EC chunks" AuditEventLogRecord = "audit event log record" + WritecacheShrinkSkippedNotEmpty = "writecache shrink skipped: database is not empty" ) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 8f37d7860..2c5e8cc3a 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -71,6 +71,7 @@ type SealWriteCachePrm struct { ShardIDs []*shard.ID IgnoreErrors bool RestoreMode bool + Shrink bool } type ShardSealResult struct { @@ -116,7 +117,7 @@ func (e *StorageEngine) SealWriteCache(ctx context.Context, prm SealWriteCachePr return nil } - err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors, RestoreMode: prm.RestoreMode}) + err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors, RestoreMode: prm.RestoreMode, Shrink: prm.Shrink}) resGuard.Lock() defer resGuard.Unlock() diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 9edad7170..c29710930 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -61,6 +61,7 @@ func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error type SealWriteCachePrm struct { IgnoreErrors bool RestoreMode bool + Shrink bool } // SealWriteCache flushes all data from the write-cache and moves it to degraded read only mode. @@ -87,5 +88,5 @@ func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error { return ErrDegradedMode } - return s.writeCache.Seal(ctx, writecache.SealPrm{IgnoreErrors: p.IgnoreErrors, RestoreMode: p.RestoreMode}) + return s.writeCache.Seal(ctx, writecache.SealPrm{IgnoreErrors: p.IgnoreErrors, RestoreMode: p.RestoreMode, Shrink: p.Shrink}) } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index da7feda9a..e34f5a76b 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -291,7 +291,7 @@ func (c *cache) Flush(ctx context.Context, ignoreErrors, seal bool) error { if seal { m := c.mode | mode.ReadOnly - if err := c.setMode(ctx, m, ignoreErrors); err != nil { + if err := c.setMode(ctx, m, setModePrm{ignoreErrors: ignoreErrors}); err != nil { return err } c.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 4172cfbc8..44da9b36e 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -2,16 +2,25 @@ package writecache import ( "context" + "errors" "fmt" + "os" + "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) +type setModePrm struct { + ignoreErrors bool + shrink bool +} + // SetMode sets write-cache mode of operation. // When shard is put in read-only mode all objects in memory are flushed to disk // and all background jobs are suspended. @@ -25,7 +34,7 @@ func (c *cache) SetMode(m mode.Mode) error { c.modeMtx.Lock() defer c.modeMtx.Unlock() - err := c.setMode(ctx, m, true) + err := c.setMode(ctx, m, setModePrm{ignoreErrors: true}) if err == nil { c.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) } @@ -33,21 +42,19 @@ func (c *cache) SetMode(m mode.Mode) error { } // setMode applies new mode. Must be called with cache.modeMtx lock taken. -func (c *cache) setMode(ctx context.Context, m mode.Mode, ignoreErrors bool) error { +func (c *cache) setMode(ctx context.Context, m mode.Mode, prm setModePrm) error { var err error turnOffMeta := m.NoMetabase() if turnOffMeta && !c.mode.NoMetabase() { - err = c.flush(ctx, ignoreErrors) + err = c.flush(ctx, prm.ignoreErrors) if err != nil { return err } } - if c.db != nil { - if err = c.db.Close(); err != nil { - return fmt.Errorf("can't close write-cache database: %w", err) - } + if err := c.closeDB(prm.shrink); err != nil { + return err } // Suspend producers to ensure there are channel send operations in fly. @@ -71,6 +78,40 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode, ignoreErrors bool) err return nil } +func (c *cache) closeDB(shrink bool) error { + if c.db == nil { + return nil + } + if !shrink { + if err := c.db.Close(); err != nil { + return fmt.Errorf("can't close write-cache database: %w", err) + } + return nil + } + + var empty bool + err := c.db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(defaultBucket) + empty = b == nil || b.Stats().KeyN == 0 + return nil + }) + if err != nil && !errors.Is(err, bbolt.ErrDatabaseNotOpen) { + return fmt.Errorf("failed to check DB items: %w", err) + } + if err := c.db.Close(); err != nil { + return fmt.Errorf("can't close write-cache database: %w", err) + } + if empty { + err := os.Remove(filepath.Join(c.path, dbName)) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove DB file: %w", err) + } + } else { + c.log.Info(logs.WritecacheShrinkSkippedNotEmpty) + } + return nil +} + // readOnly returns true if current mode is read-only. // `c.modeMtx` must be taken. func (c *cache) readOnly() bool { diff --git a/pkg/local_object_storage/writecache/seal.go b/pkg/local_object_storage/writecache/seal.go index 22b4e0988..fa224f5e0 100644 --- a/pkg/local_object_storage/writecache/seal.go +++ b/pkg/local_object_storage/writecache/seal.go @@ -22,13 +22,13 @@ func (c *cache) Seal(ctx context.Context, prm SealPrm) error { sourceMode := c.mode // flush will be done by setMode - err := c.setMode(ctx, mode.DegradedReadOnly, prm.IgnoreErrors) + err := c.setMode(ctx, mode.DegradedReadOnly, setModePrm{ignoreErrors: prm.IgnoreErrors, shrink: prm.Shrink}) if err != nil { return err } c.metrics.SetMode(mode.ComponentDisabled) if prm.RestoreMode { - err = c.setMode(ctx, sourceMode, prm.IgnoreErrors) + err = c.setMode(ctx, sourceMode, setModePrm{ignoreErrors: prm.IgnoreErrors}) if err == nil { c.metrics.SetMode(mode.ConvertToComponentMode(sourceMode)) } diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 7085a57b2..a973df604 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -23,6 +23,7 @@ type Info struct { type SealPrm struct { IgnoreErrors bool RestoreMode bool + Shrink bool } // Cache represents write-cache for objects. diff --git a/pkg/services/control/server/seal_writecache.go b/pkg/services/control/server/seal_writecache.go index b663cfc81..697b91918 100644 --- a/pkg/services/control/server/seal_writecache.go +++ b/pkg/services/control/server/seal_writecache.go @@ -20,6 +20,7 @@ func (s *Server) SealWriteCache(ctx context.Context, req *control.SealWriteCache ShardIDs: s.getShardIDList(req.GetBody().GetShard_ID()), IgnoreErrors: req.GetBody().GetIgnoreErrors(), RestoreMode: req.GetBody().GetRestoreMode(), + Shrink: req.GetBody().GetShrink(), } res, err := s.s.SealWriteCache(ctx, prm) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 895b743684054b9d9b839abb0576ebb04a20ce92..ac512f1a55f3e98a6769dca917bb873fd76e5f6d 100644 GIT binary patch delta 3103 zcmZuzdr;KZ8J+zuEUVBgwwPUacb==t(ei$&R8Yd!P;pMJCiz@hLKF`BXty_&P1n5dhTxpZJK|6vwQD% zzsLE`ckaD$*8Xt7-nr9qT+U_SUb-Y3t?M^8ZFcnqnsZ7J4;w%ZUdn6dAZ(Fwab?3^eOQFR@ z*@kPYE5A=!Rg#ioH!e?61@P|@CypK#JNC9p1x9vBjQ$&q+HUzAf7s5y*KH~aD^Ex` zc6CcU9O0@w)D}ixtFsOG!qq&Sxh~UE_0nvQ4pLOCzq9kr-zDa^(fqE)Cr2d(Bi#~# z6~`nzftKd6uQTllS4ntjn}p$fxN?du3~yAcJoxuX7RJtt)v8Z=igD;ec?Nau5{h-- zmpSlPt2C4?P#Ne5QIYV~NH(r5mpJ6TEFSdiVFg?NDw(KTDaH78fyzbyPDy~bUe;iE zikgPZA4oFpJ9*ZKQlV%JS9X*{DF;FtB@wRM#GyTkh+J@}c=%rBT7p%rgeyo@Vx?UX z`JXY`Zo6_?4E)xvGI6Dccy=z8=@_^3V8iPw7Dxm2V?l{ut#%&rPve3h}{J z1|0gXglaE;jG>oRi&Pf;StXrBPN^hZxgRfSm=sk1!)&d5v8(<)R zjtUm{3+PW(b6~HLbe`&}W^gmpyqPLWyQyqpp1p~x0z=PBn8l4Znpi5C5~2yl$=t$T z?0a5JfVCAodH)do%<7135J>^D5yZ= zUu&pxlx-BZ&OZbPJ@gxqtfruU4*Pp^J(c7$k6Hj%6({5NEHzV8z-=OqcMEvd+_<$~ z9Fr`Zi@yjda3V!b)zy|lrK)t?s#J*>TE=nSJvA znoPwje$J8+!MP^BF}U3!0qxkvao<1V3FmBL8pl1dQ?fKLEolU1f*~fZ4piRdgfNfC zrm5QW=(Ye|vLa1cb2VsjbFD#hmU1B~Q>EZaI_I2nRDh*B$l=H{lFW(R+eUW>c8~5#G?9!!);U>#k>ljuxmg6!cbdn3>Ly^dN4WOA zZ%K%y7(T6~-eH-w^trB%d7?k8H#6K%k+7m3gtl^$8cImv9z+O)NwdXS7r8>%_ z15>-*`M5P-&C)L7V0cwHhTbRrD$Wuyav=r%hoyXSETS>?3nF5cHpdk}F1Kr(&7L@Q z62>oiez7}nTUgFf9+gd zl8d98>FA3AUKkS&BJj>}c7VnV=u~AvMS3D!d&#Pso&c*jH2awP*lrGzU4O{JU@S@C ztQ{!eluTS0ap?7MuC|A8^h|(!9V~1`AxB{2n*_YGkW4pu;rJ0%XhRX%aB`6TOyWC= zNS)pVk`b6mH`u4nGm;Qq6u96UbY3Tm*dOnBfZ3luVB$T49HilLj_vO{>8xLUpUY9Wt0$~y?-Z3>>m=J-cMz@>5|ko)s%Ee53KfjDF_qg zr{KNP0I+!&FHyS#tUgH12+&|Qd{<>IM((l1AC5>tkekl|RF|V>%JIj=T(MwEV z*Dw{Lzlu87^E8bqv>_;)tI`63PdiiMD!n{ecZK#nBkb0_2c*~NRhHkGLP_#vDjNo8 zsp#NB^HQlRCsf8Kv}WvFT~8R=vq@WuOSGotBKZHtYEO)c7pyDtiP#epVA9u&-J-~- zp<=Zh*SspqtS{e<`n&Q?4Rriu1y)RXNV)va=Tx?HmE;d^THcsm(q!fx*`bRxO-f8= zT0FS8j!YZ7FK6dvVG=soJv)5CwJqYc!$tmN_2tw&Ma_iy)HBd<>DfC z|8NZfNvsL5C%;A&^5NE0qcU}>`}r4d>o?%XHG$;o=SwyW{U9r^dh|r1Vc)ZhRo6%V E1(>+I`~Uy| delta 2929 zcmZ8jc~I2%6`y?%mIW4AjzyGX5AKy?SzHk1(jX{D+?bi5#&AeIriqBDjf~QWsWC}4 zLZ^gRCkEHdq@4~EOVTuSdI*Wsn562aR+~=4Vbh6I?IbuI(_$s`^Zl)W^q-yG@4oLl zKJW8B?>l|obnBd{^RVt6`XJ*+2OkTUp=e#Sv_^LV~3)lsAcII|WGE30yCIfx={okxLCK!qzNU*rM1h-aQm3 z`;F{Kbh%XU<{~Wx?YpyCea|aU#;4ZuV`4c*!LoZF<;tD?s{v&WJc{&v z^~k3wjN^eA8j8zr3V0dW6}#9y9sz#=hhsT2Re+ z4cla>n)QQUew)tf#Cr{4BFr2pSN=nbRAb{MsS7+nq`WCwBBSc$U>9)7kK=fCY^&i! zIUWgc5iw6Z{gfhI=TNCH4rjyPC#P8dgrE0*g}nP>IaG{?sbnsEu=3Fy$BR@NNn)vG z$Kb>m9nwj}&uD2tKDsT>#j{(Q>QL1ActC8eLiiCIFP4x54ws!Cij|yNis)U7bBmp` zm2Iq)CUUws6F5pj*P%sERd9w}wR4dQI9*MNGPE9)xqmTAJhf<(={7EE*&EY2PU0Vd z$AJc_lpHG;Nt>BNM6J?M4er&Xa)e~>1GPu1xpHofLdCfcS%P%2(%4R^bJYjQYc*WF&`#5qP%)%& zjJ$aTuFhK6s7;A;yBs<|i)HK}tyURiS$S3O_u%;G0W=P$dUfi};ACk?2XiJ4fW5iv zK%~}HC}G;lE*Y+eDeSkZti$~bh{!}&o_6pCc_tJ2Z%X47F*U)?b^)FBI@ll`he7Ol zG>RMm@teM#fF(_%tjmAYq}Dn7&4$is6ASO>t}< zDpJghR6R6Hv5QZO(FykDwL^UuTuxT}*3024|71kUj1xJ74t8Z1Bk?bSiCrVWROzGm zazS_Bexv~A2Ff4u08{HjTp(j<7`&nph_&f;z&`N?N^N@wZM5_wm*K@2Rj@5YMqNnb z?g!+Mre8oR2eLUwTJoWT7|U4c&qr6iAA>;M!)OoSOq23uxH~KH`Z<%hg|Hh{2Vd}2Wz<0}xWt%M!oy9vNTD-eIy zn@c8lfdd`Id~|L+^#Z@zPF&@`Yr2GMa)WAR;0nmFu^&)q9Cwy*aG+Yzg>w5T96nxx zaHTgv-l3KJuylVxdPAPtx6XE#+(U#16?|~~IV5oSfNLyV$EJm9qrm-A4zw%4P z9s?Axdk@H>F)%3CA9m5)y>#W>DArNZ9st*;&~49OP}RJ6Xyv!B228%QfCRy`x4>*! zm16h?9Cm(&BpV+BU%lywIh)tKf4zcrgKS|qn*2N2d)0+gYtZRASakUkxe0SUd--{p zzDYrnUxC(N8o`FrhT`%X1hJN4QHvq)PQ_xDQG4 zd?+=;4|d?Mw@4cR08pWt5dT2ykFWK&-nmUYVIVJFOM2-G$NI%u3DRv7-7m4sut$w^ zuPLX5e+q&EbiKHhD$lOk|1$}dvR$w%N=2`KTn(`-t-*-*q#`$MTP=9hB*~DOjsC^& zNrMD6JqO7s>j6ZL8t{JMqs6pLdsr^ZQ!ha`o|n>6%Wg%;6SbVFv&zW*pLg+7j?|)@ e+Byzb29MX_WMd`3&8;Gg@((Cy@E>*DbM=32gP>sm diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index a10410025..486f30a93 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -658,6 +658,9 @@ message SealWriteCacheRequest { // If true, then writecache will be sealed, but mode will be restored to the current one. bool restore_mode = 4; + + // If true, then writecache will shrink internal storage. + bool shrink = 5; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index b6b064973d2a677fcf842e83b01a6930c7f2151a..417d25c05e9e47799ddcd960120a7ae1c31e5295 100644 GIT binary patch delta 88 zcmexylKtu__J%EtpSMkJ78B<-)lsO>3(hFY%*)<9(P0YX^gl-#1*YFWz{omXe;Xt3 p=7y^>jFTNCd8VJ+$*4HJWGAD{R5eCH4w#+r{`p3jkG)BYXe= delta 85 zcmcb5iv7n)_J%EtpSNunO=g_PIKANvqrmh72N+qW>u+P^-P~|hhHMo|p_K$Rn`