Compare commits

...

4 commits

Author SHA1 Message Date
8bbd0d1687 [#1331] *: Remove outdated TODO
Signed-off-by: Ekaterina Lebedeva <ekaterina.lebedeva@yadro.com>
2024-09-09 19:36:19 +03:00
e04305b3aa [#1331] tree: Add order flag to tree get-subtree
Added `--ordered` flag to sort output by ascending FileName.

Signed-off-by: Ekaterina Lebedeva <ekaterina.lebedeva@yadro.com>
2024-09-09 19:36:19 +03:00
4668efc0bf [#1355] metabase: Upgrade improvements
Do not fail on same latest version to run compact on upgraded metabase.
Use NoSync on compact.
Log every batch on bucket delete stage.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-09-09 11:37:36 +03:00
654d970fad [#1355] adm: Run metabase upgrade concurrently
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-09-06 15:54:40 +03:00
5 changed files with 45 additions and 15 deletions

View file

@ -3,6 +3,7 @@ package metabase
import (
"errors"
"fmt"
"sync"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags"
@ -11,6 +12,7 @@ import (
shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"github.com/spf13/cobra"
"golang.org/x/sync/errgroup"
)
const (
@ -57,17 +59,28 @@ func upgrade(cmd *cobra.Command, _ []string) error {
cmd.Println(i+1, ":", path)
}
result := make(map[string]bool)
var resultGuard sync.Mutex
eg, ctx := errgroup.WithContext(cmd.Context())
for _, path := range paths {
cmd.Println("upgrading metabase", path, "...")
if err := meta.Upgrade(cmd.Context(), path, !noCompact, func(a ...any) {
cmd.Println(append([]any{time.Now().Format(time.RFC3339), ":", path, ":"}, a...)...)
}); err != nil {
result[path] = false
cmd.Println("error: failed to upgrade metabase", path, ":", err)
} else {
result[path] = true
cmd.Println("metabase", path, "upgraded successfully")
}
eg.Go(func() error {
var success bool
cmd.Println("upgrading metabase", path, "...")
if err := meta.Upgrade(ctx, path, !noCompact, func(a ...any) {
cmd.Println(append([]any{time.Now().Format(time.RFC3339), ":", path, ":"}, a...)...)
}); err != nil {
cmd.Println("error: failed to upgrade metabase", path, ":", err)
} else {
success = true
cmd.Println("metabase", path, "upgraded successfully")
}
resultGuard.Lock()
result[path] = success
resultGuard.Unlock()
return nil
})
}
if err := eg.Wait(); err != nil {
return err
}
for mb, ok := range result {
if ok {

View file

@ -86,8 +86,6 @@ var listContainersCmd = &cobra.Command{
if flagVarListPrintAttr {
cnr.IterateAttributes(func(key, val string) {
if !strings.HasPrefix(key, container.SysAttributePrefix) && !strings.HasPrefix(key, container.SysAttributePrefixNeoFS) {
// FIXME(@cthulhu-rider): https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/97
// Use dedicated method to skip system attributes.
cmd.Printf(" %s: %s\n", key, val)
}
})

View file

@ -49,6 +49,7 @@ const (
heightFlagKey = "height"
countFlagKey = "count"
depthFlagKey = "depth"
orderFlagKey = "ordered"
)
func initCTID(cmd *cobra.Command) {

View file

@ -30,6 +30,7 @@ func initGetSubtreeCmd() {
ff := getSubtreeCmd.Flags()
ff.Uint64(rootIDFlagKey, 0, "Root ID to traverse from.")
ff.Uint32(depthFlagKey, 10, "Traversal depth.")
ff.Bool(orderFlagKey, false, "Sort output by ascending FileName.")
_ = getSubtreeCmd.MarkFlagRequired(commonflags.CIDFlag)
_ = getSubtreeCmd.MarkFlagRequired(treeIDFlagKey)
@ -59,6 +60,13 @@ func getSubTree(cmd *cobra.Command, _ []string) {
depth, _ := cmd.Flags().GetUint32(depthFlagKey)
order, _ := cmd.Flags().GetBool(orderFlagKey)
bodyOrder := tree.GetSubTreeRequest_Body_Order_None
if order {
bodyOrder = tree.GetSubTreeRequest_Body_Order_Asc
}
var bt []byte
if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil {
bt = t.Marshal()
@ -71,6 +79,9 @@ func getSubTree(cmd *cobra.Command, _ []string) {
RootId: []uint64{rid},
Depth: depth,
BearerToken: bt,
OrderBy: &tree.GetSubTreeRequest_Body_Order{
Direction: bodyOrder,
},
},
}

View file

@ -27,6 +27,10 @@ const (
var updates = map[uint64]func(ctx context.Context, db *bbolt.DB, log func(a ...any)) error{
2: upgradeFromV2ToV3,
3: func(_ context.Context, _ *bbolt.DB, log func(a ...any)) error {
log("metabase already upgraded")
return nil
},
}
func Upgrade(ctx context.Context, path string, compact bool, log func(a ...any)) error {
@ -86,6 +90,7 @@ func compactDB(db *bbolt.DB) error {
}
dst, err := bbolt.Open(tmpFileName, f.Mode(), &bbolt.Options{
Timeout: 100 * time.Millisecond,
NoSync: true,
})
if err != nil {
return fmt.Errorf("can't open new metabase to compact: %w", err)
@ -93,6 +98,9 @@ func compactDB(db *bbolt.DB) error {
if err := bbolt.Compact(dst, db, compactMaxTxSize); err != nil {
return fmt.Errorf("compact metabase: %w", errors.Join(err, dst.Close(), os.Remove(tmpFileName)))
}
if err := dst.Sync(); err != nil {
return fmt.Errorf("sync compacted metabase: %w", errors.Join(err, os.Remove(tmpFileName)))
}
if err := dst.Close(); err != nil {
return fmt.Errorf("close compacted metabase: %w", errors.Join(err, os.Remove(tmpFileName)))
}
@ -369,8 +377,7 @@ func dropBucketsByPrefix(ctx context.Context, db *bbolt.DB, prefix []byte, log f
log("deleting buckets completed with an error:", err)
return err
}
if count += uint64(len(keys)); count%upgradeLogFrequency == 0 {
log("deleted", count, "buckets")
}
count += uint64(len(keys))
log("deleted", count, "buckets")
}
}