From 5f1c7d0f0017be7c6d665e59b3f0756d2321f62a Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 15 Jul 2024 14:07:32 +0300 Subject: [PATCH 1/7] [#1223] scripts: Add script to populate metabase Signed-off-by: Aleksey Savchuk --- .../populate-metabase/internal/generate.go | 132 +++++++++ .../populate-metabase/internal/populate.go | 263 ++++++++++++++++++ scripts/populate-metabase/main.go | 159 +++++++++++ 3 files changed, 554 insertions(+) create mode 100644 scripts/populate-metabase/internal/generate.go create mode 100644 scripts/populate-metabase/internal/populate.go create mode 100644 scripts/populate-metabase/main.go diff --git a/scripts/populate-metabase/internal/generate.go b/scripts/populate-metabase/internal/generate.go new file mode 100644 index 000000000..d2004b673 --- /dev/null +++ b/scripts/populate-metabase/internal/generate.go @@ -0,0 +1,132 @@ +package internal + +import ( + "crypto/sha256" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "git.frostfs.info/TrueCloudLab/tzhash/tz" + "golang.org/x/exp/rand" +) + +func GeneratePayloadPool(count uint, size uint) [][]byte { + pool := [][]byte{} + for i := uint(0); i < count; i++ { + payload := make([]byte, size) + _, _ = rand.Read(payload) + + pool = append(pool, payload) + } + return pool +} + +func GenerateAttributePool(count uint) []objectSDK.Attribute { + pool := []objectSDK.Attribute{} + for i := uint(0); i < count; i++ { + for j := uint(0); j < count; j++ { + attr := *objectSDK.NewAttribute() + attr.SetKey(fmt.Sprintf("key%d", i)) + attr.SetValue(fmt.Sprintf("value%d", j)) + pool = append(pool, attr) + } + } + return pool +} + +func GenerateOwnerPool(count uint) []user.ID { + pool := []user.ID{} + for i := uint(0); i < count; i++ { + pool = append(pool, usertest.ID()) + } + return pool +} + +type ObjectOption func(obj *objectSDK.Object) + +func GenerateObject(options ...ObjectOption) *objectSDK.Object { + var ver version.Version + ver.SetMajor(2) + ver.SetMinor(1) + + payload := make([]byte, 0) + + var csum checksum.Checksum + csum.SetSHA256(sha256.Sum256(payload)) + + var csumTZ checksum.Checksum + csumTZ.SetTillichZemor(tz.Sum(csum.Value())) + + obj := objectSDK.New() + obj.SetID(oidtest.ID()) + obj.SetOwnerID(usertest.ID()) + obj.SetContainerID(cidtest.ID()) + + header := objecttest.Object().GetECHeader() + header.SetParent(oidtest.ID()) + obj.SetECHeader(header) + + obj.SetVersion(&ver) + obj.SetPayload(payload) + obj.SetPayloadSize(uint64(len(payload))) + obj.SetPayloadChecksum(csum) + obj.SetPayloadHomomorphicHash(csumTZ) + + for _, option := range options { + option(obj) + } + + return obj +} + +func WithContainerID(cid cid.ID) ObjectOption { + return func(obj *objectSDK.Object) { + obj.SetContainerID(cid) + } +} + +func WithType(typ objectSDK.Type) ObjectOption { + return func(obj *objectSDK.Object) { + obj.SetType(typ) + } +} + +func WithPayloadFromPool(pool [][]byte) ObjectOption { + payload := pool[rand.Intn(len(pool))] + + var csum checksum.Checksum + csum.SetSHA256(sha256.Sum256(payload)) + + var csumTZ checksum.Checksum + csumTZ.SetTillichZemor(tz.Sum(csum.Value())) + + return func(obj *objectSDK.Object) { + obj.SetPayload(payload) + obj.SetPayloadSize(uint64(len(payload))) + obj.SetPayloadChecksum(csum) + obj.SetPayloadHomomorphicHash(csumTZ) + } +} + +func WithAttributesFromPool(pool []objectSDK.Attribute, count uint) ObjectOption { + return func(obj *objectSDK.Object) { + attrs := []objectSDK.Attribute{} + for i := uint(0); i < count; i++ { + attrs = append(attrs, pool[rand.Intn(len(pool))]) + } + obj.SetAttributes(attrs...) + } +} + +func WithOwnerIDFromPool(pool []user.ID) ObjectOption { + return func(obj *objectSDK.Object) { + obj.SetOwnerID(pool[rand.Intn(len(pool))]) + } +} diff --git a/scripts/populate-metabase/internal/populate.go b/scripts/populate-metabase/internal/populate.go new file mode 100644 index 000000000..390c1cdc0 --- /dev/null +++ b/scripts/populate-metabase/internal/populate.go @@ -0,0 +1,263 @@ +package internal + +import ( + "context" + "fmt" + "math/rand" + "sync" + + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "golang.org/x/sync/errgroup" +) + +type EpochState struct{} + +func (s EpochState) CurrentEpoch() uint64 { + return 0 +} + +func PopulateWithObjects( + ctx context.Context, + db *meta.DB, + group *errgroup.Group, + count uint, + factory func() *objectSDK.Object, +) { + digits := "0123456789" + + for i := uint(0); i < count; i++ { + obj := factory() + + id := []byte(fmt.Sprintf( + "%c/%c/%c", + digits[rand.Int()%len(digits)], + digits[rand.Int()%len(digits)], + digits[rand.Int()%len(digits)], + )) + + prm := meta.PutPrm{} + prm.SetObject(obj) + prm.SetStorageID(id) + + group.Go(func() error { + if _, err := db.Put(ctx, prm); err != nil { + return fmt.Errorf("couldn't put an object: %w", err) + } + return nil + }) + } +} + +func PopulateWithBigObjects( + ctx context.Context, + db *meta.DB, + group *errgroup.Group, + count uint, + factory func() *objectSDK.Object, +) { + for i := uint(0); i < count; i++ { + group.Go(func() error { + if err := populateWithBigObject(ctx, db, factory); err != nil { + return fmt.Errorf("couldn't put a big object: %w", err) + } + return nil + }) + } +} + +func populateWithBigObject( + ctx context.Context, + db *meta.DB, + factory func() *objectSDK.Object, +) error { + t := &target{db: db} + + pk, _ := keys.NewPrivateKey() + p := transformer.NewPayloadSizeLimiter(transformer.Params{ + Key: &pk.PrivateKey, + NextTargetInit: func() transformer.ObjectWriter { return t }, + NetworkState: EpochState{}, + MaxSize: 10, + }) + + obj := factory() + payload := make([]byte, 30) + + err := p.WriteHeader(ctx, obj) + if err != nil { + return err + } + + _, err = p.Write(ctx, payload) + if err != nil { + return err + } + + _, err = p.Close(ctx) + if err != nil { + return err + } + + return nil +} + +type target struct { + db *meta.DB +} + +func (t *target) WriteObject(ctx context.Context, obj *objectSDK.Object) error { + prm := meta.PutPrm{} + prm.SetObject(obj) + + _, err := t.db.Put(ctx, prm) + return err +} + +func PopulateGraveyard( + ctx context.Context, + db *meta.DB, + group *errgroup.Group, + workBufferSize int, + count uint, + factory func() *objectSDK.Object, +) { + ts := factory() + ts.SetType(objectSDK.TypeTombstone) + + prm := meta.PutPrm{} + prm.SetObject(ts) + + group.Go(func() error { + if _, err := db.Put(ctx, prm); err != nil { + return fmt.Errorf("couldn't put a tombstone object: %w", err) + } + return nil + }) + + cID, _ := ts.ContainerID() + oID, _ := ts.ID() + + var tsAddr oid.Address + + tsAddr.SetContainer(cID) + tsAddr.SetObject(oID) + + addrs := make(chan oid.Address, workBufferSize) + + go func() { + defer close(addrs) + + wg := &sync.WaitGroup{} + wg.Add(int(count)) + + for i := uint(0); i < count; i++ { + obj := factory() + + prm := meta.PutPrm{} + prm.SetObject(obj) + + group.Go(func() error { + defer wg.Done() + + if _, err := db.Put(ctx, prm); err != nil { + return fmt.Errorf("couldn't put an object: %w", err) + } + + cID, _ := obj.ContainerID() + oID, _ := obj.ID() + + var addr oid.Address + addr.SetContainer(cID) + addr.SetObject(oID) + + addrs <- addr + return nil + }) + } + wg.Wait() + }() + + go func() { + for addr := range addrs { + prm := meta.InhumePrm{} + prm.SetAddresses(addr) + prm.SetTombstoneAddress(tsAddr) + + group.Go(func() error { + if _, err := db.Inhume(ctx, prm); err != nil { + return fmt.Errorf("couldn't inhume an object: %w", err) + } + return nil + }) + } + }() +} + +func PopulateLocked( + ctx context.Context, + db *meta.DB, + group *errgroup.Group, + workBufferSize int, + count uint, + factory func() *objectSDK.Object, +) { + locker := factory() + locker.SetType(objectSDK.TypeLock) + + prm := meta.PutPrm{} + prm.SetObject(locker) + + group.Go(func() error { + if _, err := db.Put(ctx, prm); err != nil { + return fmt.Errorf("couldn't put a locker object: %w", err) + } + return nil + }) + + ids := make(chan oid.ID, workBufferSize) + + go func() { + defer close(ids) + + wg := &sync.WaitGroup{} + wg.Add(int(count)) + + for i := uint(0); i < count; i++ { + defer wg.Done() + + obj := factory() + + prm := meta.PutPrm{} + prm.SetObject(obj) + + group.Go(func() error { + if _, err := db.Put(ctx, prm); err != nil { + return fmt.Errorf("couldn't put an object: %w", err) + } + + id, _ := obj.ID() + ids <- id + return nil + }) + } + wg.Wait() + }() + + go func() { + for id := range ids { + lockerCID, _ := locker.ContainerID() + lockerOID, _ := locker.ID() + + group.Go(func() error { + if err := db.Lock(ctx, lockerCID, lockerOID, []oid.ID{id}); err != nil { + return fmt.Errorf("couldn't lock an object: %w", err) + } + return nil + }) + } + }() +} diff --git a/scripts/populate-metabase/main.go b/scripts/populate-metabase/main.go new file mode 100644 index 000000000..2bc7a5553 --- /dev/null +++ b/scripts/populate-metabase/main.go @@ -0,0 +1,159 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/scripts/populate-metabase/internal" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "golang.org/x/sync/errgroup" +) + +var ( + path string + force bool + jobs uint + + numContainers, + numObjects, + numAttributesPerObj, + numOwners, + numPayloads, + numAttributes uint +) + +func main() { + flag.StringVar(&path, "path", "", "Path to metabase") + flag.BoolVar(&force, "force", false, "Rewrite existing database") + flag.UintVar(&jobs, "j", 10000, "Number of jobs to run") + + flag.UintVar(&numContainers, "containers", 0, "Number of containers to be created") + flag.UintVar(&numObjects, "objects", 0, "Number of objects per container") + flag.UintVar(&numAttributesPerObj, "attributes", 0, "Number of attributes per object") + + flag.UintVar(&numOwners, "distinct-owners", 10, "Number of distinct owners to be used") + flag.UintVar(&numPayloads, "distinct-payloads", 10, "Number of distinct payloads to be used") + flag.UintVar(&numAttributes, "distinct-attributes", 10, "Number of distinct attributes to be used") + + flag.Parse() + + exitIf(numPayloads == 0, "must have payloads\n") + exitIf(numAttributes == 0, "must have attributes\n") + exitIf(numOwners == 0, "must have owners\n") + exitIf(len(path) == 0, "path to metabase not specified\n") + exitIf( + numAttributesPerObj > numAttributes, + "object can't have more attributes than available\n", + ) + + info, err := os.Stat(path) + exitIf( + err != nil && !errors.Is(err, os.ErrNotExist), + "couldn't get path info: %s\n", err, + ) + + // Path exits. + if err == nil { + exitIf(info.IsDir(), "path is a directory\n") + exitIf(!force, "couldn't rewrite existing file, use '-force' flag\n") + + err = os.Remove(path) + exitIf(err != nil, "couldn't remove existing file: %s\n", err) + } + + err = populate() + exitIf(err != nil, "couldn't populate the metabase: %s\n", err) +} + +func getObjectFactory(opts ...internal.ObjectOption) func() *objectSDK.Object { + return func() *objectSDK.Object { + return internal.GenerateObject(opts...) + } +} + +func populate() (err error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + db := meta.New([]meta.Option{ + meta.WithPath(path), + meta.WithPermissions(0o600), + meta.WithEpochState(internal.EpochState{}), + }...) + + if err = db.Open(ctx, mode.ReadWrite); err != nil { + return fmt.Errorf("couldn't open the metabase: %w", err) + } + defer func() { + if errOnClose := db.Close(); errOnClose != nil { + err = errors.Join( + err, + fmt.Errorf("couldn't close the metabase: %w", db.Close()), + ) + } + }() + + if err = db.Init(); err != nil { + return fmt.Errorf("couldn't init the metabase: %w", err) + } + + payloads := internal.GeneratePayloadPool(numPayloads, 32) + attributes := internal.GenerateAttributePool(numAttributes) + owners := internal.GenerateOwnerPool(numOwners) + + types := []objectSDK.Type{ + objectSDK.TypeRegular, + objectSDK.TypeLock, + objectSDK.TypeTombstone, + } + + eg, ctx := errgroup.WithContext(ctx) + eg.SetLimit(int(jobs)) + + for i := uint(0); i < numContainers; i++ { + cid := cidtest.ID() + + for _, typ := range types { + internal.PopulateWithObjects(ctx, db, eg, numObjects, getObjectFactory( + internal.WithContainerID(cid), + internal.WithType(typ), + internal.WithPayloadFromPool(payloads), + internal.WithOwnerIDFromPool(owners), + internal.WithAttributesFromPool(attributes, numAttributesPerObj), + )) + } + internal.PopulateWithBigObjects(ctx, db, eg, numObjects, getObjectFactory( + internal.WithContainerID(cid), + internal.WithType(objectSDK.TypeRegular), + internal.WithAttributesFromPool(attributes, numAttributesPerObj), + internal.WithOwnerIDFromPool(owners), + )) + internal.PopulateGraveyard(ctx, db, eg, int(jobs), numObjects, getObjectFactory( + internal.WithContainerID(cid), + internal.WithType(objectSDK.TypeRegular), + internal.WithAttributesFromPool(attributes, numAttributesPerObj), + internal.WithOwnerIDFromPool(owners), + )) + internal.PopulateLocked(ctx, db, eg, int(jobs), numObjects, getObjectFactory( + internal.WithContainerID(cid), + internal.WithType(objectSDK.TypeRegular), + internal.WithAttributesFromPool(attributes, numAttributesPerObj), + internal.WithOwnerIDFromPool(owners), + )) + } + + return eg.Wait() +} + +func exitIf(cond bool, format string, args ...any) { + if cond { + fmt.Fprintf(os.Stderr, format, args...) + os.Exit(1) + } +} -- 2.45.2 From a3a9f42e076dd7e69d3957e43fb3e38b201ec0c1 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 18 Jul 2024 18:26:11 +0300 Subject: [PATCH 2/7] [#1223] lens/tui: Add metabase schema Signed-off-by: Aleksey Savchuk --- .../internal/schema/common/format.go | 43 +++ .../internal/schema/common/raw.go | 29 ++ .../internal/schema/common/schema.go | 81 ++++++ .../schema/metabase/buckets/detailed.go | 29 ++ .../schema/metabase/buckets/filter.go | 81 ++++++ .../schema/metabase/buckets/parsers.go | 111 ++++++++ .../schema/metabase/buckets/prefix.go | 53 ++++ .../schema/metabase/buckets/string.go | 48 ++++ .../internal/schema/metabase/buckets/types.go | 166 ++++++++++++ .../internal/schema/metabase/parser.go | 29 ++ .../schema/metabase/records/detailed.go | 65 +++++ .../schema/metabase/records/filter.go | 145 ++++++++++ .../schema/metabase/records/parsers.go | 251 ++++++++++++++++++ .../schema/metabase/records/string.go | 135 ++++++++++ .../internal/schema/metabase/records/types.go | 82 ++++++ .../internal/schema/metabase/records/util.go | 20 ++ go.mod | 5 +- go.sum | Bin 39703 -> 40482 bytes 18 files changed, 1372 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-lens/internal/schema/common/format.go create mode 100644 cmd/frostfs-lens/internal/schema/common/raw.go create mode 100644 cmd/frostfs-lens/internal/schema/common/schema.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/detailed.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/filter.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/string.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/types.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/parser.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/detailed.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/filter.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/parsers.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/string.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/types.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/util.go diff --git a/cmd/frostfs-lens/internal/schema/common/format.go b/cmd/frostfs-lens/internal/schema/common/format.go new file mode 100644 index 000000000..4ed7e96f2 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/common/format.go @@ -0,0 +1,43 @@ +package common + +import ( + "fmt" + "strconv" + + "github.com/gdamore/tcell/v2" +) + +type FormatOptions struct { + Color tcell.Color + + Bold, + Italic, + Underline, + StrikeThrough bool +} + +func Format(s string, opts FormatOptions) string { + var boldTag, italicTag, underlineTag, strikeThroughTag string + + switch { + case opts.Bold: + boldTag = "b" + case opts.Italic: + italicTag = "i" + case opts.Underline: + underlineTag = "u" + case opts.StrikeThrough: + strikeThroughTag = "s" + } + + attrs := fmt.Sprintf( + "%s%s%s%s", boldTag, italicTag, underlineTag, strikeThroughTag, + ) + color := strconv.FormatInt(int64(opts.Color.Hex()), 16) + + return fmt.Sprintf("[#%06s::%s]%s[-::-]", color, attrs, s) +} + +func FormatSimple(s string, c tcell.Color) string { + return Format(s, FormatOptions{Color: c}) +} diff --git a/cmd/frostfs-lens/internal/schema/common/raw.go b/cmd/frostfs-lens/internal/schema/common/raw.go new file mode 100644 index 000000000..0990e24c3 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/common/raw.go @@ -0,0 +1,29 @@ +package common + +import ( + "github.com/davecgh/go-spew/spew" + "github.com/gdamore/tcell/v2" + "github.com/mr-tron/base58" +) + +type RawEntry struct { + key, value []byte +} + +var RawParser Parser = rawParser + +func rawParser(key, value []byte) (SchemaEntry, Parser, error) { + return &RawEntry{key: key, value: value}, rawParser, nil +} + +func (r *RawEntry) String() string { + return FormatSimple(base58.Encode(r.key), tcell.ColorRed) +} + +func (r *RawEntry) DetailedString() string { + return spew.Sdump(r) +} + +func (r *RawEntry) Filter(string, any) FilterResult { + return No +} diff --git a/cmd/frostfs-lens/internal/schema/common/schema.go b/cmd/frostfs-lens/internal/schema/common/schema.go new file mode 100644 index 000000000..9bad19032 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/common/schema.go @@ -0,0 +1,81 @@ +package common + +import ( + "errors" + "fmt" +) + +type FilterResult byte + +const ( + No FilterResult = iota + Maybe + Yes +) + +func IfThenElse(condition bool, onSuccess, onFailure FilterResult) FilterResult { + var res FilterResult + if condition { + res = onSuccess + } else { + res = onFailure + } + return res +} + +type SchemaEntry interface { + String() string + DetailedString() string + Filter(typ string, val any) FilterResult +} + +type ( + Parser func(key, value []byte) (SchemaEntry, Parser, error) + FallbackParser func(key, value []byte) (SchemaEntry, Parser) +) + +func Any(parsers ...Parser) Parser { + return func(key, value []byte) (SchemaEntry, Parser, error) { + var errs error + for _, parser := range parsers { + ret, next, err := parser(key, value) + if err == nil { + return ret, next, nil + } + errs = errors.Join(errs, err) + } + return nil, nil, fmt.Errorf("no parser succeeded: %w", errs) + } +} + +func WithFallback(parser Parser, fallback FallbackParser) Parser { + if parser == nil { + return fallback.ToParser() + } + return func(key, value []byte) (SchemaEntry, Parser, error) { + entry, next, err := parser(key, value) + if err == nil { + return entry, WithFallback(next, fallback), nil + } + return fallback.ToParser()(key, value) + } +} + +func (fp FallbackParser) ToParser() Parser { + return func(key, value []byte) (SchemaEntry, Parser, error) { + entry, next := fp(key, value) + return entry, next, nil + } +} + +func (p Parser) ToFallbackParser() FallbackParser { + return func(key, value []byte) (SchemaEntry, Parser) { + entry, next, err := p(key, value) + if err != nil { + panic(fmt.Errorf( + "couldn't use that parser as a fallback parser, it returned an error: %w", err, + )) + } + return entry, next + } +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/detailed.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/detailed.go new file mode 100644 index 000000000..6a08a723e --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/detailed.go @@ -0,0 +1,29 @@ +package buckets + +import ( + "github.com/davecgh/go-spew/spew" +) + +func (b *PrefixBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *PrefixContainerBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *UserBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *ContainerBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *UserAttributeKeyBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *UserAttributeValueBucket) DetailedString() string { + return spew.Sdump(*b) +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/filter.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/filter.go new file mode 100644 index 000000000..891c4004f --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/filter.go @@ -0,0 +1,81 @@ +package buckets + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" +) + +func (b *PrefixBucket) Filter(typ string, _ any) common.FilterResult { + switch typ { + case "cid": + return b.resolvers.cidResolver(false) + case "oid": + return b.resolvers.oidResolver(false) + default: + return common.No + } +} + +func (b *PrefixContainerBucket) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return b.resolvers.cidResolver(b.id.Equals(id)) + case "oid": + return b.resolvers.oidResolver(false) + default: + return common.No + } +} + +func (b *UserBucket) Filter(typ string, _ any) common.FilterResult { + switch typ { + case "cid": + return b.resolvers.cidResolver(false) + case "oid": + return b.resolvers.oidResolver(false) + default: + return common.No + } +} + +func (b *ContainerBucket) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return b.resolvers.cidResolver(b.id.Equals(id)) + case "oid": + return b.resolvers.oidResolver(false) + default: + return common.No + } +} + +func (b *UserAttributeKeyBucket) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(b.id.Equals(id), common.Yes, common.No) + case "oid": + return common.Maybe + case "key": + key := val.(string) + return common.IfThenElse(b.key == key, common.Yes, common.No) + case "value": + return common.Maybe + default: + return common.No + } +} + +func (b *UserAttributeValueBucket) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + return common.Maybe + case "value": + value := val.(string) + return common.IfThenElse(b.value == value, common.Yes, common.No) + default: + return common.No + } +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go new file mode 100644 index 000000000..24cc0e52d --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go @@ -0,0 +1,111 @@ +package buckets + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase/records" +) + +var ( + GraveyardParser = NewPrefixBucketParser(Graveyard, records.GraveyardRecordParser, Resolvers{ + cidResolver: LenientResolver, + oidResolver: LenientResolver, + }) + + GarbageParser = NewPrefixBucketParser(Garbage, records.GarbageRecordParser, Resolvers{ + cidResolver: LenientResolver, + oidResolver: LenientResolver, + }) + + ContainerVolumeParser = NewPrefixBucketParser(ContainerVolume, records.ContainerVolumeRecordParser, Resolvers{ + cidResolver: LenientResolver, + oidResolver: StrictResolver, + }) + + LockedParser = NewPrefixBucketParser( + Locked, + NewContainerBucketParser( + records.LockedRecordParser, + Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }, + ), + Resolvers{ + cidResolver: LenientResolver, + oidResolver: LenientResolver, + }, + ) + + ShardInfoParser = NewPrefixBucketParser(ShardInfo, records.ShardInfoRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: StrictResolver, + }) + + PrimaryParser = NewPrefixContainerBucketParser(Primary, records.ObjectRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + LockersParser = NewPrefixContainerBucketParser(Lockers, records.ObjectRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + TombstoneParser = NewPrefixContainerBucketParser(Tombstone, records.ObjectRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + SmallParser = NewPrefixContainerBucketParser(Small, records.SmallRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + RootParser = NewPrefixContainerBucketParser(Root, records.RootRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + OwnerParser = NewPrefixContainerBucketParser( + Owner, + NewUserBucketParser( + records.OwnerRecordParser, + Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }, + ), + Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }, + ) + + UserAttributeParser = NewUserAttributeKeyBucketParser( + NewUserAttributeValueBucketParser(records.UserAttributeRecordParser), + ) + + PayloadHashParser = NewPrefixContainerBucketParser(PayloadHash, records.PayloadHashRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: StrictResolver, + }) + + ParentParser = NewPrefixContainerBucketParser(Parent, records.ParentRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + SplitParser = NewPrefixContainerBucketParser(Split, records.SplitRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: StrictResolver, + }) + + ContainerCountersParser = NewPrefixBucketParser(ContainerCounters, records.ContainerCountersRecordParser, Resolvers{ + cidResolver: LenientResolver, + oidResolver: StrictResolver, + }) + + ECInfoParser = NewPrefixContainerBucketParser(ECInfo, records.ECInfoRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) +) diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go new file mode 100644 index 000000000..2fb122940 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go @@ -0,0 +1,53 @@ +package buckets + +type Prefix byte + +const ( + Graveyard Prefix = iota + Garbage + ToMoveIt + ContainerVolume + Locked + ShardInfo + Primary + Lockers + _ + Tombstone + Small + Root + Owner + UserAttribute + PayloadHash + Parent + Split + ContainerCounters + ECInfo +) + +var x = map[Prefix]string{ + Graveyard: "Graveyard", + Garbage: "Garbage", + ToMoveIt: "To Move It", + ContainerVolume: "Container Volume", + Locked: "Locked", + ShardInfo: "Shard Info", + Primary: "Primary", + Lockers: "Lockers", + Tombstone: "Tombstone", + Small: "Small", + Root: "Root", + Owner: "Owner", + UserAttribute: "User Attribute", + PayloadHash: "Payload Hash", + Parent: "Parent", + Split: "Split", + ContainerCounters: "Container Counters", + ECInfo: "EC Info", +} + +func (p Prefix) String() string { + if s, ok := x[p]; ok { + return s + } + return "Unknown Prefix" +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/string.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/string.go new file mode 100644 index 000000000..db90bddbd --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/string.go @@ -0,0 +1,48 @@ +package buckets + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "github.com/gdamore/tcell/v2" +) + +func (b *PrefixBucket) String() string { + return common.FormatSimple( + fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime, + ) +} + +func (b *PrefixContainerBucket) String() string { + return fmt.Sprintf( + "%s CID %s", + common.FormatSimple( + fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime, + ), + common.FormatSimple(b.id.String(), tcell.ColorAqua), + ) +} + +func (b *UserBucket) String() string { + return "UID " + common.FormatSimple(b.id.String(), tcell.ColorAqua) +} + +func (b *ContainerBucket) String() string { + return "CID " + common.FormatSimple(b.id.String(), tcell.ColorAqua) +} + +func (b *UserAttributeKeyBucket) String() string { + return fmt.Sprintf("%s CID %s ATTR-KEY %s", + common.FormatSimple( + fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime, + ), + common.FormatSimple( + fmt.Sprintf("%-44s", b.id), tcell.ColorAqua, + ), + common.FormatSimple(b.key, tcell.ColorAqua), + ) +} + +func (b *UserAttributeValueBucket) String() string { + return "ATTR-VALUE " + common.FormatSimple(b.value, tcell.ColorAqua) +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/types.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/types.go new file mode 100644 index 000000000..82b47dd85 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/types.go @@ -0,0 +1,166 @@ +package buckets + +import ( + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/mr-tron/base58" +) + +type ( + PrefixBucket struct { + prefix Prefix + resolvers Resolvers + } + + PrefixContainerBucket struct { + prefix Prefix + id cid.ID + resolvers Resolvers + } + + ContainerBucket struct { + id cid.ID + resolvers Resolvers + } + + UserBucket struct { + id user.ID + resolvers Resolvers + } + + UserAttributeKeyBucket struct { + prefix Prefix + id cid.ID + key string + } + + UserAttributeValueBucket struct { + value string + } +) + +type ( + FilterResolver = func(result bool) common.FilterResult + + Resolvers struct { + cidResolver FilterResolver + oidResolver FilterResolver + } +) + +var ( + StrictResolver = func(x bool) common.FilterResult { return common.IfThenElse(x, common.Yes, common.No) } + LenientResolver = func(x bool) common.FilterResult { return common.IfThenElse(x, common.Yes, common.Maybe) } +) + +var ( + ErrNotBucket = errors.New("not a bucket") + ErrInvalidKeyLength = errors.New("invalid key length") + ErrInvalidValueLength = errors.New("invalid value length") + ErrInvalidPrefix = errors.New("invalid prefix") +) + +func NewPrefixBucketParser(prefix Prefix, next common.Parser, resolvers Resolvers) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + if len(key) != 1 { + return nil, nil, ErrInvalidKeyLength + } + var b PrefixBucket + if b.prefix = Prefix(key[0]); b.prefix != prefix { + return nil, nil, ErrInvalidPrefix + } + b.resolvers = resolvers + return &b, next, nil + } +} + +func NewPrefixContainerBucketParser(prefix Prefix, next common.Parser, resolvers Resolvers) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + if len(key) != 33 { + return nil, nil, ErrInvalidKeyLength + } + var b PrefixContainerBucket + if b.prefix = Prefix(key[0]); b.prefix != prefix { + return nil, nil, ErrInvalidPrefix + } + if err := b.id.Decode(key[1:]); err != nil { + return nil, nil, err + } + b.resolvers = resolvers + return &b, next, nil + } +} + +func NewUserBucketParser(next common.Parser, resolvers Resolvers) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + var b UserBucket + if err := b.id.DecodeString(base58.Encode(key)); err != nil { + return nil, nil, err + } + b.resolvers = resolvers + return &b, next, nil + } +} + +func NewContainerBucketParser(next common.Parser, resolvers Resolvers) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + if len(key) != 32 { + return nil, nil, ErrInvalidKeyLength + } + var b ContainerBucket + if err := b.id.Decode(key); err != nil { + return nil, nil, err + } + b.resolvers = resolvers + return &b, next, nil + } +} + +func NewUserAttributeKeyBucketParser(next common.Parser) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + if len(key) < 34 { + return nil, nil, ErrInvalidKeyLength + } + var b UserAttributeKeyBucket + if b.prefix = Prefix(key[0]); b.prefix != UserAttribute { + return nil, nil, ErrInvalidPrefix + } + if err := b.id.Decode(key[1:33]); err != nil { + return nil, nil, err + } + b.key = string(key[33:]) + return &b, next, nil + } +} + +func NewUserAttributeValueBucketParser(next common.Parser) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + if len(key) == 0 { + return nil, nil, ErrInvalidKeyLength + } + var b UserAttributeValueBucket + b.value = string(key) + return &b, next, nil + } +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/parser.go b/cmd/frostfs-lens/internal/schema/metabase/parser.go new file mode 100644 index 000000000..ea095e207 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/parser.go @@ -0,0 +1,29 @@ +package metabase + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase/buckets" +) + +var MetabaseParser = common.WithFallback( + common.Any( + buckets.GraveyardParser, + buckets.GarbageParser, + buckets.ContainerVolumeParser, + buckets.LockedParser, + buckets.ShardInfoParser, + buckets.PrimaryParser, + buckets.LockersParser, + buckets.TombstoneParser, + buckets.SmallParser, + buckets.RootParser, + buckets.OwnerParser, + buckets.UserAttributeParser, + buckets.PayloadHashParser, + buckets.ParentParser, + buckets.SplitParser, + buckets.ContainerCountersParser, + buckets.ECInfoParser, + ), + common.RawParser.ToFallbackParser(), +) diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/detailed.go b/cmd/frostfs-lens/internal/schema/metabase/records/detailed.go new file mode 100644 index 000000000..2dda15b4f --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/detailed.go @@ -0,0 +1,65 @@ +package records + +import ( + "github.com/davecgh/go-spew/spew" +) + +func (r *GraveyardRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *GarbageRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ContainerVolumeRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *LockedRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ShardInfoRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ObjectRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *SmallRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *RootRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *OwnerRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *UserAttributeRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *PayloadHashRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ParentRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *SplitRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ContainerCountersRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ECInfoRecord) DetailedString() string { + return spew.Sdump(*r) +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/filter.go b/cmd/frostfs-lens/internal/schema/metabase/records/filter.go new file mode 100644 index 000000000..880a7a8ff --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/filter.go @@ -0,0 +1,145 @@ +package records + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +func (r *GraveyardRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.object.Container().Equals(id), common.Yes, common.No) + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.object.Object().Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *GarbageRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.addr.Container().Equals(id), common.Yes, common.No) + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.addr.Object().Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *ContainerVolumeRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *ShardInfoRecord) Filter(string, any) common.FilterResult { + return common.No +} + +func (r *LockedRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *ObjectRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *SmallRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *RootRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *OwnerRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *UserAttributeRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *PayloadHashRecord) Filter(string, any) common.FilterResult { + return common.No +} + +func (r *ParentRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.parent.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *SplitRecord) Filter(string, any) common.FilterResult { + return common.No +} + +func (r *ContainerCountersRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *ECInfoRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/parsers.go b/cmd/frostfs-lens/internal/schema/metabase/records/parsers.go new file mode 100644 index 000000000..1b070e2a0 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/parsers.go @@ -0,0 +1,251 @@ +package records + +import ( + "encoding/binary" + "errors" + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +var ( + ErrInvalidKeyLength = errors.New("invalid key length") + ErrInvalidValueLength = errors.New("invalid value length") + ErrInvalidPrefix = errors.New("invalid prefix") +) + +func GraveyardRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 64 { + return nil, nil, ErrInvalidKeyLength + } + if len(value) != 64 { + return nil, nil, ErrInvalidValueLength + } + var ( + cnr cid.ID + obj oid.ID + r GraveyardRecord + ) + + _ = cnr.Decode(key[:32]) + _ = obj.Decode(key[32:]) + + r.object.SetContainer(cnr) + r.object.SetObject(obj) + + _ = cnr.Decode(value[:32]) + _ = obj.Decode(value[32:]) + + r.tombstone.SetContainer(cnr) + r.tombstone.SetObject(obj) + + return &r, nil, nil +} + +func GarbageRecordParser(key, _ []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 64 { + return nil, nil, ErrInvalidKeyLength + } + var ( + cnr cid.ID + obj oid.ID + r GarbageRecord + ) + + _ = cnr.Decode(key[:32]) + _ = obj.Decode(key[32:]) + + r.addr.SetContainer(cnr) + r.addr.SetObject(obj) + + return &r, nil, nil +} + +func ContainerVolumeRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 32 { + return nil, nil, ErrInvalidKeyLength + } + if len(value) != 8 { + return nil, nil, ErrInvalidValueLength + } + var r ContainerVolumeRecord + + _ = r.id.Decode(key) + r.volume = binary.LittleEndian.Uint64(value) + + return &r, nil, nil +} + +func LockedRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var ( + r LockedRecord + err error + ) + + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + if r.ids, err = DecodeOIDs(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func ShardInfoRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) == 0 { + return nil, nil, ErrInvalidKeyLength + } + + var r ShardInfoRecord + if string(key) == "id" { + r.label = string(key) + r.value = shard.ID(value).String() + + return &r, nil, nil + } + + if len(value) != 8 { + return nil, nil, ErrInvalidValueLength + } + r.label = string(key) + r.value = strconv.FormatUint(binary.LittleEndian.Uint64(value), 10) + + return &r, nil, nil +} + +func ObjectRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 32 { + return nil, nil, ErrInvalidKeyLength + } + var r ObjectRecord + + _ = r.id.Decode(key) + if err := r.object.Unmarshal(value); err != nil { + return nil, nil, err + } + + return &r, nil, nil +} + +func SmallRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var r SmallRecord + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + if len(value) != 0 { + x := string(value) + r.storageID = &x + } + return &r, nil, nil +} + +func RootRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var r RootRecord + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + if len(value) == 0 { + return &r, nil, nil + } + r.info = &objectSDK.SplitInfo{} + if err := r.info.Unmarshal(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func OwnerRecordParser(key, _ []byte) (common.SchemaEntry, common.Parser, error) { + var r OwnerRecord + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func UserAttributeRecordParser(key, _ []byte) (common.SchemaEntry, common.Parser, error) { + var r UserAttributeRecord + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func PayloadHashRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 32 { + return nil, nil, ErrInvalidKeyLength + } + var ( + err error + r PayloadHashRecord + ) + + r.checksum.SetSHA256([32]byte(key)) + if r.ids, err = DecodeOIDs(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func ParentRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var ( + r ParentRecord + err error + ) + if err = r.parent.Decode(key); err != nil { + return nil, nil, err + } + if r.ids, err = DecodeOIDs(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func SplitRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var ( + err error + r SplitRecord + ) + if err = r.id.UnmarshalBinary(key); err != nil { + return nil, nil, err + } + if r.ids, err = DecodeOIDs(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func ContainerCountersRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(value) != 24 { + return nil, nil, ErrInvalidValueLength + } + + var r ContainerCountersRecord + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + + r.logical = binary.LittleEndian.Uint64(value[:8]) + r.physical = binary.LittleEndian.Uint64(value[8:16]) + r.user = binary.LittleEndian.Uint64(value[16:24]) + + return &r, nil, nil +} + +func ECInfoRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var ( + r ECInfoRecord + err error + ) + + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + if r.ids, err = DecodeOIDs(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/string.go b/cmd/frostfs-lens/internal/schema/metabase/records/string.go new file mode 100644 index 000000000..a6c70d537 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/string.go @@ -0,0 +1,135 @@ +package records + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +func (r *GraveyardRecord) String() string { + return fmt.Sprintf( + "Object CID %s OID %s %c Tombstone CID %s OID %s", + common.FormatSimple(fmt.Sprintf("%-44s", r.object.Container()), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.object.Object()), tcell.ColorAqua), + tview.Borders.Vertical, + common.FormatSimple(fmt.Sprintf("%-44s", r.tombstone.Container()), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.tombstone.Object()), tcell.ColorAqua), + ) +} + +func (r *GarbageRecord) String() string { + return fmt.Sprintf( + "CID %-44s OID %-44s", + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Container()), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Object()), tcell.ColorAqua), + ) +} + +func (r *ContainerVolumeRecord) String() string { + return fmt.Sprintf( + "CID %-44s %c %d", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + r.volume, + ) +} + +func (r *LockedRecord) String() string { + return fmt.Sprintf( + "Locker OID %s %c Locked [%d]OID {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + len(r.ids), + ) +} + +func (r *ShardInfoRecord) String() string { + return fmt.Sprintf("%-13s %c %s", r.label, tview.Borders.Vertical, r.value) +} + +func (r *ObjectRecord) String() string { + return fmt.Sprintf( + "OID %s %c Object {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + ) +} + +func (r *SmallRecord) String() string { + s := fmt.Sprintf( + "OID %s %c", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + ) + if r.storageID != nil { + s = fmt.Sprintf("%s %s", s, *r.storageID) + } + return s +} + +func (r *RootRecord) String() string { + s := fmt.Sprintf( + "Root OID %s %c", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + ) + if r.info != nil { + s += " Split info {...}" + } + return s +} + +func (r *OwnerRecord) String() string { + return "OID " + common.FormatSimple(r.id.String(), tcell.ColorAqua) +} + +func (r *UserAttributeRecord) String() string { + return "OID " + common.FormatSimple(r.id.String(), tcell.ColorAqua) +} + +func (r *PayloadHashRecord) String() string { + return fmt.Sprintf( + "Checksum %s %c [%d]OID {...}", + common.FormatSimple(r.checksum.String(), tcell.ColorAqua), + tview.Borders.Vertical, + len(r.ids), + ) +} + +func (r *ParentRecord) String() string { + return fmt.Sprintf( + "Parent OID %s %c [%d]OID {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.parent), tcell.ColorAqua), + tview.Borders.Vertical, + len(r.ids), + ) +} + +func (r *SplitRecord) String() string { + return fmt.Sprintf( + "Split ID %s %c [%d]OID {...}", + common.FormatSimple(r.id.String(), tcell.ColorAqua), + tview.Borders.Vertical, + len(r.ids), + ) +} + +func (r *ContainerCountersRecord) String() string { + return fmt.Sprintf( + "CID %s %c logical %d, physical %d, user %d", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + r.logical, r.physical, r.user, + ) +} + +func (r *ECInfoRecord) String() string { + return fmt.Sprintf( + "OID %s %c [%d]OID {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + len(r.ids), + ) +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/types.go b/cmd/frostfs-lens/internal/schema/metabase/records/types.go new file mode 100644 index 000000000..34c1c29fd --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/types.go @@ -0,0 +1,82 @@ +package records + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/google/uuid" +) + +type ( + GraveyardRecord struct { + object, tombstone oid.Address + } + + GarbageRecord struct { + addr oid.Address + } + + ContainerVolumeRecord struct { + id cid.ID + volume uint64 + } + + LockedRecord struct { + id oid.ID + ids []oid.ID + } + + ShardInfoRecord struct { + label string + value string + } + + ObjectRecord struct { + id oid.ID + object objectSDK.Object + } + + SmallRecord struct { + id oid.ID + storageID *string // optional + } + + RootRecord struct { + id oid.ID + info *objectSDK.SplitInfo // optional + } + + OwnerRecord struct { + id oid.ID + } + + UserAttributeRecord struct { + id oid.ID + } + + PayloadHashRecord struct { + checksum checksum.Checksum + ids []oid.ID + } + + ParentRecord struct { + parent oid.ID + ids []oid.ID + } + + SplitRecord struct { + id uuid.UUID + ids []oid.ID + } + + ContainerCountersRecord struct { + id cid.ID + logical, physical, user uint64 + } + + ECInfoRecord struct { + id oid.ID + ids []oid.ID + } +) diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/util.go b/cmd/frostfs-lens/internal/schema/metabase/records/util.go new file mode 100644 index 000000000..f50ebe951 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/util.go @@ -0,0 +1,20 @@ +package records + +import ( + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/nspcc-dev/neo-go/pkg/io" +) + +func DecodeOIDs(data []byte) ([]oid.ID, error) { + r := io.NewBinReaderFromBuf(data) + + size := r.ReadVarUint() + oids := make([]oid.ID, size) + + for i := uint64(0); i < size; i++ { + if err := oids[i].Decode(r.ReadVarBytes()); err != nil { + return nil, err + } + } + return oids, nil +} diff --git a/go.mod b/go.mod index 358370201..927d978ac 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,9 @@ require ( github.com/VictoriaMetrics/easyproto v0.1.4 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 + github.com/gdamore/tcell/v2 v2.7.4 github.com/go-pkgz/expirable-cache/v3 v3.0.0 github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 @@ -65,10 +67,10 @@ require ( github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/gdamore/encoding v1.0.0 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -85,6 +87,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/klauspost/reedsolomon v1.12.1 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/minio/sha256-simd v1.0.1 // indirect diff --git a/go.sum b/go.sum index be82bff70b53e73c47603ed223fa440bdc0a88df..3fb24b3903f35783a6d27f2f7175b43100ea7a6c 100644 GIT binary patch delta 623 zcmaLT%Zk%b0Dxia(!D4MrMQzYxi!t13NCsvbtboI(lkwW$w``f&h+G@GwG%v`UG9L z)`j3sEVvNG=b4Q=_gWMwGWY@?KmY4bt+!uVAKtaMfma|LGGPf(kd#qp&Cx8)c4F=c z1#~p!hENP=MXyY(&Cp{9CJU$sTUqa3Xsknu3c(9^q7LgLz|Au5>Yc9Mw@U7+9~ zd%3})yemMVzm-PjP)TD{U2ovU^Qph&2_9z%^@5z5<7d&WG=X8BGi$!H=4n6+2eXPq z#wurOKb>5<7KJ2dP>sEK31fR8*{6$ZzLAnrGY5`!Olp5~|1pnp19L$ZGgD^gmCgCd zmDn4*ipy&<1}VF`VAyk@Yej`UJKEi}AMHQ4Z|-mN8hVv~5;h_JMV&yDBfPBh!(aaQ zU#=`KCpwNB*R8c?S##4=POP)h6bMvT(ngxtAEcrT%FH<~X8te#>1Ga33?VQP(T%>W z60dO*E(E~Y)ay{D5`gPXGg%RmWSP>iyYJ%T{Wnp1c&EFY8RPwrX^U%dVE_Kb$JQyJ x*`pn~Vhq!>c^$^4@8+IPCaMqB63!@Zr6@`v<0{p#71dX}H!mOT-B(}l{RD2j)t3MO delta 38 wcmV+>0NMYdyaJcD0RPr#hv4vy;vu23A6+hV!oB#j- -- 2.45.2 From 85e255a3ca89f33ea687ef63b077f9374b83020b Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 16 Aug 2024 14:33:03 +0300 Subject: [PATCH 3/7] [#1223] lens/tui: Add writecache schema Signed-off-by: Aleksey Savchuk --- .../internal/schema/writecache/parsers.go | 63 ++++++++++++++++++ .../internal/schema/writecache/types.go | 66 +++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 cmd/frostfs-lens/internal/schema/writecache/parsers.go create mode 100644 cmd/frostfs-lens/internal/schema/writecache/types.go diff --git a/cmd/frostfs-lens/internal/schema/writecache/parsers.go b/cmd/frostfs-lens/internal/schema/writecache/parsers.go new file mode 100644 index 000000000..7d70b27b2 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/writecache/parsers.go @@ -0,0 +1,63 @@ +package writecache + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/mr-tron/base58" +) + +var WritecacheParser = common.WithFallback( + DefaultBucketParser, + common.RawParser.ToFallbackParser(), +) + +func DefaultBucketParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, errors.New("not a bucket") + } + if !bytes.Equal(key, []byte{0}) { + return nil, nil, errors.New("invalid key") + } + return &DefaultBucket{}, DefaultRecordParser, nil +} + +func DefaultRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + parts := strings.Split(string(key), "/") + + if len(parts) != 2 { + return nil, nil, errors.New("invalid key, expected address string /") + } + + cnrRaw, err := base58.Decode(parts[0]) + if err != nil { + return nil, nil, errors.New("can't decode CID string") + } + objRaw, err := base58.Decode(parts[1]) + if err != nil { + return nil, nil, errors.New("can't decode OID string") + } + + cnr := cid.ID{} + if err := cnr.Decode(cnrRaw); err != nil { + return nil, nil, fmt.Errorf("can't decode CID: %w", err) + } + obj := oid.ID{} + if err := obj.Decode(objRaw); err != nil { + return nil, nil, fmt.Errorf("can't decode OID: %w", err) + } + + var r DefaultRecord + + r.addr.SetContainer(cnr) + r.addr.SetObject(obj) + + r.data = value[:] + + return &r, nil, nil +} diff --git a/cmd/frostfs-lens/internal/schema/writecache/types.go b/cmd/frostfs-lens/internal/schema/writecache/types.go new file mode 100644 index 000000000..3f71c5366 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/writecache/types.go @@ -0,0 +1,66 @@ +package writecache + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/davecgh/go-spew/spew" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type ( + DefaultBucket struct{} + + DefaultRecord struct { + addr oid.Address + data []byte + } +) + +func (b *DefaultBucket) String() string { + return common.FormatSimple("0 Default", tcell.ColorLime) +} + +func (r *DefaultRecord) String() string { + return fmt.Sprintf( + "CID %s OID %s %c Data {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Container()), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Object()), tcell.ColorAqua), + tview.Borders.Vertical, + ) +} + +func (b *DefaultBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (r *DefaultRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (b *DefaultBucket) Filter(typ string, _ any) common.FilterResult { + switch typ { + case "cid": + return common.Maybe + case "oid": + return common.Maybe + default: + return common.No + } +} + +func (r *DefaultRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.addr.Container().Equals(id), common.Yes, common.No) + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.addr.Object().Equals(id), common.Yes, common.No) + default: + return common.No + } +} -- 2.45.2 From e40dc9a199aac45057d76a9c5aafa8a1d6e3080a Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 11 Jul 2024 19:39:54 +0300 Subject: [PATCH 4/7] [#1223] lens/tui: Add TUI app to explore metabase Signed-off-by: Aleksey Savchuk --- cmd/frostfs-lens/internal/meta/root.go | 1 + cmd/frostfs-lens/internal/meta/tui.go | 82 ++++ cmd/frostfs-lens/internal/tui/buckets.go | 257 ++++++++++ cmd/frostfs-lens/internal/tui/db.go | 160 +++++++ cmd/frostfs-lens/internal/tui/detailed.go | 24 + cmd/frostfs-lens/internal/tui/filter.go | 44 ++ cmd/frostfs-lens/internal/tui/input.go | 77 +++ cmd/frostfs-lens/internal/tui/loading.go | 72 +++ cmd/frostfs-lens/internal/tui/records.go | 271 +++++++++++ cmd/frostfs-lens/internal/tui/types.go | 18 + cmd/frostfs-lens/internal/tui/ui.go | 548 ++++++++++++++++++++++ cmd/frostfs-lens/internal/tui/util.go | 97 ++++ go.mod | 3 +- go.sum | Bin 40482 -> 40699 bytes 14 files changed, 1653 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-lens/internal/meta/tui.go create mode 100644 cmd/frostfs-lens/internal/tui/buckets.go create mode 100644 cmd/frostfs-lens/internal/tui/db.go create mode 100644 cmd/frostfs-lens/internal/tui/detailed.go create mode 100644 cmd/frostfs-lens/internal/tui/filter.go create mode 100644 cmd/frostfs-lens/internal/tui/input.go create mode 100644 cmd/frostfs-lens/internal/tui/loading.go create mode 100644 cmd/frostfs-lens/internal/tui/records.go create mode 100644 cmd/frostfs-lens/internal/tui/types.go create mode 100644 cmd/frostfs-lens/internal/tui/ui.go create mode 100644 cmd/frostfs-lens/internal/tui/util.go diff --git a/cmd/frostfs-lens/internal/meta/root.go b/cmd/frostfs-lens/internal/meta/root.go index 6741abd0c..351d1ce80 100644 --- a/cmd/frostfs-lens/internal/meta/root.go +++ b/cmd/frostfs-lens/internal/meta/root.go @@ -32,6 +32,7 @@ func init() { inspectCMD, listGraveyardCMD, listGarbageCMD, + tuiCMD, ) } diff --git a/cmd/frostfs-lens/internal/meta/tui.go b/cmd/frostfs-lens/internal/meta/tui.go new file mode 100644 index 000000000..00e8bf117 --- /dev/null +++ b/cmd/frostfs-lens/internal/meta/tui.go @@ -0,0 +1,82 @@ +package meta + +import ( + "context" + "fmt" + + common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" + schema "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui" + "github.com/rivo/tview" + "github.com/spf13/cobra" + "go.etcd.io/bbolt" +) + +var tuiCMD = &cobra.Command{ + Use: "explore", + Short: "Metabase exploration with a terminal UI", + Long: `Launch a terminal UI to explore metabase and search for data. + +Available search filters: +- cid CID +- oid OID +- addr CID/OID +- attr key[/value] +`, + Run: tuiFunc, +} + +var initialPrompt string + +func init() { + common.AddComponentPathFlag(tuiCMD, &vPath) + + tuiCMD.Flags().StringVar( + &initialPrompt, + "filter", + "", + "Filter prompt to start with, format 'tag:value [+ tag:value]...'", + ) +} + +func tuiFunc(cmd *cobra.Command, _ []string) { + common.ExitOnErr(cmd, runTUI(cmd)) +} + +func runTUI(cmd *cobra.Command) error { + db, err := openDB(false) + if err != nil { + return fmt.Errorf("couldn't open database: %w", err) + } + defer db.Close() + + // Need if app was stopped with Ctrl-C. + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() + + app := tview.NewApplication() + ui := tui.NewUI(ctx, app, db, schema.MetabaseParser, nil) + + _ = ui.AddFilter("cid", tui.CIDParser, "CID") + _ = ui.AddFilter("oid", tui.OIDParser, "OID") + _ = ui.AddCompositeFilter("addr", tui.AddressParser, "CID/OID") + _ = ui.AddCompositeFilter("attr", tui.AttributeParser, "key[/value]") + + err = ui.WithPrompt(initialPrompt) + if err != nil { + return fmt.Errorf("invalid filter prompt: %w", err) + } + + app.SetRoot(ui, true).SetFocus(ui) + return app.Run() +} + +func openDB(writable bool) (*bbolt.DB, error) { + db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{ + ReadOnly: !writable, + }) + if err != nil { + return nil, err + } + return db, nil +} diff --git a/cmd/frostfs-lens/internal/tui/buckets.go b/cmd/frostfs-lens/internal/tui/buckets.go new file mode 100644 index 000000000..3f5088e7a --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/buckets.go @@ -0,0 +1,257 @@ +package tui + +import ( + "context" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type BucketsView struct { + *tview.Box + + mu sync.Mutex + + view *tview.TreeView + nodeToUpdate *tview.TreeNode + + ui *UI + filter *Filter +} + +type bucketNode struct { + bucket *Bucket + filter *Filter +} + +func NewBucketsView(ui *UI, filter *Filter) *BucketsView { + return &BucketsView{ + Box: tview.NewBox(), + view: tview.NewTreeView(), + ui: ui, + filter: filter, + } +} + +func (v *BucketsView) Mount(_ context.Context) error { + root := tview.NewTreeNode(".") + root.SetExpanded(false) + root.SetSelectable(false) + root.SetReference(&bucketNode{ + bucket: &Bucket{NextParser: v.ui.rootParser}, + filter: v.filter, + }) + + v.nodeToUpdate = root + + v.view.SetRoot(root) + v.view.SetCurrentNode(root) + + return nil +} + +func (v *BucketsView) Update(ctx context.Context) error { + if v.nodeToUpdate == nil { + return nil + } + defer func() { v.nodeToUpdate = nil }() + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + ready := make(chan struct{}) + errCh := make(chan error) + + tmp := tview.NewTreeNode(v.nodeToUpdate.GetText()) + tmp.SetReference(v.nodeToUpdate.GetReference()) + + node := v.nodeToUpdate.GetReference().(*bucketNode) + + go func() { + defer close(ready) + + hasBuckets, err := HasBuckets(ctx, v.ui.db, node.bucket.Path) + if err != nil { + errCh <- err + } + + // Show the selected bucket's records instead. + if !hasBuckets && node.bucket.NextParser != nil { + v.ui.moveNextPage(NewRecordsView(v.ui, node.bucket, node.filter)) + } + + if v.nodeToUpdate.IsExpanded() { + return + } + + err = v.loadNodeChildren(ctx, tmp, node.filter) + if err != nil { + errCh <- err + } + }() + + select { + case <-ctx.Done(): + case <-ready: + v.mu.Lock() + v.nodeToUpdate.SetChildren(tmp.GetChildren()) + v.nodeToUpdate.SetExpanded(!v.nodeToUpdate.IsExpanded()) + v.mu.Unlock() + case err := <-errCh: + return err + } + + return nil +} + +func (v *BucketsView) Unmount() { +} + +func (v *BucketsView) Draw(screen tcell.Screen) { + x, y, width, height := v.GetInnerRect() + v.view.SetRect(x, y, width, height) + + v.view.Draw(screen) +} + +func (v *BucketsView) loadNodeChildren( + ctx context.Context, node *tview.TreeNode, filter *Filter, +) error { + parentBucket := node.GetReference().(*bucketNode).bucket + + path := parentBucket.Path + parser := parentBucket.NextParser + + buffer, err := LoadBuckets(ctx, v.ui.db, path, v.ui.loadBufferSize) + if err != nil { + return err + } + + for item := range buffer { + if item.err != nil { + return item.err + } + bucket := item.val + + bucket.Entry, bucket.NextParser, err = parser(bucket.Name, nil) + if err != nil { + return err + } + + satisfies, err := v.bucketSatisfiesFilter(ctx, bucket, filter) + if err != nil { + return err + } + if !satisfies { + continue + } + + child := tview.NewTreeNode(bucket.Entry.String()). + SetSelectable(true). + SetExpanded(false). + SetReference(&bucketNode{ + bucket: bucket, + filter: filter.Apply(bucket.Entry), + }) + + node.AddChild(child) + } + + return nil +} + +func (v *BucketsView) bucketSatisfiesFilter( + ctx context.Context, bucket *Bucket, filter *Filter, +) (bool, error) { + // Does the current bucket satisfies the filter? + filter = filter.Apply(bucket.Entry) + + if filter.Result() == common.Yes { + return true, nil + } + + if filter.Result() == common.No { + return false, nil + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Check the current bucket's nested buckets if exist + bucketsBuffer, err := LoadBuckets(ctx, v.ui.db, bucket.Path, v.ui.loadBufferSize) + if err != nil { + return false, err + } + + for item := range bucketsBuffer { + if item.err != nil { + return false, item.err + } + b := item.val + + b.Entry, b.NextParser, err = bucket.NextParser(b.Name, nil) + if err != nil { + return false, err + } + + satisfies, err := v.bucketSatisfiesFilter(ctx, b, filter) + if err != nil { + return false, err + } + if satisfies { + return true, nil + } + } + + // Check the current bucket's nested records if exist + recordsBuffer, err := LoadRecords(ctx, v.ui.db, bucket.Path, v.ui.loadBufferSize) + if err != nil { + return false, err + } + + for item := range recordsBuffer { + if item.err != nil { + return false, item.err + } + r := item.val + + r.Entry, _, err = bucket.NextParser(r.Key, r.Value) + if err != nil { + return false, err + } + + if filter.Apply(r.Entry).Result() == common.Yes { + return true, nil + } + } + + return false, nil +} + +func (v *BucketsView) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) { + return v.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) { + currentNode := v.view.GetCurrentNode() + if currentNode == nil { + return + } + + switch event.Key() { + case tcell.KeyEnter: + // Expand or collapse the selected bucket's nested buckets, + // otherwise, navigate to that bucket's records. + v.nodeToUpdate = currentNode + case tcell.KeyCtrlR: + // Navigate to the selected bucket's records. + bucketNode := currentNode.GetReference().(*bucketNode) + v.ui.moveNextPage(NewRecordsView(v.ui, bucketNode.bucket, bucketNode.filter)) + case tcell.KeyCtrlD: + // Navigate to the selected bucket's detailed view. + bucketNode := currentNode.GetReference().(*bucketNode) + v.ui.moveNextPage(NewDetailedView(bucketNode.bucket.Entry.DetailedString())) + default: + v.view.InputHandler()(event, func(tview.Primitive) {}) + } + }) +} diff --git a/cmd/frostfs-lens/internal/tui/db.go b/cmd/frostfs-lens/internal/tui/db.go new file mode 100644 index 000000000..d0cf611d4 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/db.go @@ -0,0 +1,160 @@ +package tui + +import ( + "context" + "errors" + "fmt" + + "go.etcd.io/bbolt" +) + +type Item[T any] struct { + val T + err error +} + +func resolvePath(tx *bbolt.Tx, path [][]byte) (*bbolt.Bucket, error) { + if len(path) == 0 { + return nil, errors.New("can't find bucket without path") + } + + name := path[0] + bucket := tx.Bucket(name) + if bucket == nil { + return nil, fmt.Errorf("no bucket with name %s", name) + } + for _, name := range path[1:] { + bucket = bucket.Bucket(name) + if bucket == nil { + return nil, fmt.Errorf("no bucket with name %s", name) + } + } + return bucket, nil +} + +func load[T any]( + ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int, + filter func(key, value []byte) bool, transform func(key, value []byte) T, +) (<-chan Item[T], error) { + buffer := make(chan Item[T], bufferSize) + + go func() { + defer close(buffer) + + err := db.View(func(tx *bbolt.Tx) error { + var cursor *bbolt.Cursor + if len(path) == 0 { + cursor = tx.Cursor() + } else { + bucket, err := resolvePath(tx, path) + if err != nil { + buffer <- Item[T]{err: fmt.Errorf("can't find bucket: %w", err)} + return nil + } + cursor = bucket.Cursor() + } + + key, value := cursor.First() + for { + if key == nil { + return nil + } + if filter != nil && !filter(key, value) { + key, value = cursor.Next() + continue + } + + select { + case <-ctx.Done(): + return nil + case buffer <- Item[T]{val: transform(key, value)}: + key, value = cursor.Next() + } + } + }) + if err != nil { + buffer <- Item[T]{err: err} + } + }() + + return buffer, nil +} + +func LoadBuckets( + ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int, +) (<-chan Item[*Bucket], error) { + buffer, err := load( + ctx, db, path, bufferSize, + func(_, value []byte) bool { + return value == nil + }, + func(key, _ []byte) *Bucket { + base := make([][]byte, 0, len(path)) + base = append(base, path...) + + return &Bucket{ + Name: key, + Path: append(base, key), + } + }, + ) + if err != nil { + return nil, fmt.Errorf("can't start iterating bucket: %w", err) + } + + return buffer, nil +} + +func LoadRecords( + ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int, +) (<-chan Item[*Record], error) { + buffer, err := load( + ctx, db, path, bufferSize, + func(_, value []byte) bool { + return value != nil + }, + func(key, value []byte) *Record { + base := make([][]byte, 0, len(path)) + base = append(base, path...) + + return &Record{ + Key: key, + Value: value, + Path: append(base, key), + } + }, + ) + if err != nil { + return nil, fmt.Errorf("can't start iterating bucket: %w", err) + } + + return buffer, nil +} + +// HasBuckets checks if a bucket has nested buckets. It relies on assumption +// that a bucket can have either nested buckets or records but not both. +func HasBuckets(ctx context.Context, db *bbolt.DB, path [][]byte) (bool, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + buffer, err := load( + ctx, db, path, 1, + nil, + func(_, value []byte) []byte { return value }, + ) + if err != nil { + return false, err + } + + x, ok := <-buffer + if !ok { + return false, nil + } + if x.err != nil { + return false, err + } + if x.val != nil { + return false, err + } + return true, nil +} diff --git a/cmd/frostfs-lens/internal/tui/detailed.go b/cmd/frostfs-lens/internal/tui/detailed.go new file mode 100644 index 000000000..b2d897230 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/detailed.go @@ -0,0 +1,24 @@ +package tui + +import ( + "context" + + "github.com/rivo/tview" +) + +type DetailedView struct { + *tview.TextView +} + +func NewDetailedView(detailed string) *DetailedView { + v := &DetailedView{ + TextView: tview.NewTextView(), + } + v.SetDynamicColors(true) + v.SetText(detailed) + return v +} + +func (v *DetailedView) Mount(_ context.Context) error { return nil } +func (v *DetailedView) Update(_ context.Context) error { return nil } +func (v *DetailedView) Unmount() {} diff --git a/cmd/frostfs-lens/internal/tui/filter.go b/cmd/frostfs-lens/internal/tui/filter.go new file mode 100644 index 000000000..e7879eca7 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/filter.go @@ -0,0 +1,44 @@ +package tui + +import ( + "maps" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" +) + +type Filter struct { + values map[string]any + results map[string]common.FilterResult +} + +func NewFilter(values map[string]any) *Filter { + f := &Filter{ + values: maps.Clone(values), + results: make(map[string]common.FilterResult), + } + for tag := range values { + f.results[tag] = common.No + } + return f +} + +func (f *Filter) Apply(e common.SchemaEntry) *Filter { + filter := &Filter{ + values: f.values, + results: maps.Clone(f.results), + } + + for tag, value := range filter.values { + filter.results[tag] = max(filter.results[tag], e.Filter(tag, value)) + } + + return filter +} + +func (f *Filter) Result() common.FilterResult { + current := common.Yes + for _, r := range f.results { + current = min(r, current) + } + return current +} diff --git a/cmd/frostfs-lens/internal/tui/input.go b/cmd/frostfs-lens/internal/tui/input.go new file mode 100644 index 000000000..4fdf97119 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/input.go @@ -0,0 +1,77 @@ +package tui + +import ( + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type InputFieldWithHistory struct { + *tview.InputField + history []string + historyLimit int + historyPointer int + currentContent string +} + +func NewInputFieldWithHistory(historyLimit int) *InputFieldWithHistory { + return &InputFieldWithHistory{ + InputField: tview.NewInputField(), + historyLimit: historyLimit, + } +} + +func (f *InputFieldWithHistory) AddToHistory(s string) { + // Stop scrolling history on history change, need to start scrolling again. + defer func() { f.historyPointer = len(f.history) }() + + // Used history data for search prompt, so just make that data recent. + if f.historyPointer != len(f.history) && s == f.history[f.historyPointer] { + f.history = append(f.history[:f.historyPointer], f.history[f.historyPointer+1:]...) + f.history = append(f.history, s) + } + + if len(f.history) == f.historyLimit { + f.history = f.history[1:] + } + f.history = append(f.history, s) +} + +func (f *InputFieldWithHistory) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) { + return f.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) { + switch event.Key() { + case tcell.KeyDown: + if len(f.history) == 0 { + return + } + // Need to start iterating before. + if f.historyPointer == len(f.history) { + return + } + // Iterate to most recent prompts. + f.historyPointer++ + // Stop iterating over history. + if f.historyPointer == len(f.history) { + f.InputField.SetText(f.currentContent) + return + } + f.InputField.SetText(f.history[f.historyPointer]) + case tcell.KeyUp: + if len(f.history) == 0 { + return + } + // Start iterating over history. + if f.historyPointer == len(f.history) { + f.currentContent = f.InputField.GetText() + } + // End of history. + if f.historyPointer == 0 { + return + } + // Iterate to least recent prompts. + f.historyPointer-- + f.InputField.SetText(f.history[f.historyPointer]) + default: + f.InputField.InputHandler()(event, func(tview.Primitive) {}) + } + }) +} diff --git a/cmd/frostfs-lens/internal/tui/loading.go b/cmd/frostfs-lens/internal/tui/loading.go new file mode 100644 index 000000000..4b9384ad4 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/loading.go @@ -0,0 +1,72 @@ +package tui + +import ( + "context" + "fmt" + "sync/atomic" + "time" + + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type LoadingBar struct { + *tview.Box + view *tview.TextView + secondsElapsed atomic.Int64 + needDrawFunc func() + reset func() +} + +func NewLoadingBar(needDrawFunc func()) *LoadingBar { + b := &LoadingBar{ + Box: tview.NewBox(), + view: tview.NewTextView(), + needDrawFunc: needDrawFunc, + } + b.view.SetBackgroundColor(tview.Styles.PrimaryTextColor) + b.view.SetTextColor(b.GetBackgroundColor()) + + return b +} + +func (b *LoadingBar) Start(ctx context.Context) { + ctx, b.reset = context.WithCancel(ctx) + + go func() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + b.secondsElapsed.Store(0) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + b.secondsElapsed.Add(1) + b.needDrawFunc() + } + } + }() +} + +func (b *LoadingBar) Stop() { + b.reset() +} + +func (b *LoadingBar) Draw(screen tcell.Screen) { + seconds := b.secondsElapsed.Load() + + var time string + switch { + case seconds < 60: + time = fmt.Sprintf("%ds", seconds) + default: + time = fmt.Sprintf("%dm%ds", seconds/60, seconds%60) + } + b.view.SetText(fmt.Sprintf(" Loading... %s (press Escape to cancel) ", time)) + + x, y, width, _ := b.GetInnerRect() + b.view.SetRect(x, y, width, 1) + b.view.Draw(screen) +} diff --git a/cmd/frostfs-lens/internal/tui/records.go b/cmd/frostfs-lens/internal/tui/records.go new file mode 100644 index 000000000..5f53ed287 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/records.go @@ -0,0 +1,271 @@ +package tui + +import ( + "context" + "errors" + "fmt" + "math" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type updateType int + +const ( + other updateType = iota + moveToPrevPage + moveToNextPage + moveUp + moveDown + moveHome + moveEnd +) + +type RecordsView struct { + *tview.Box + + mu sync.RWMutex + + onUnmount func() + + bucket *Bucket + records []*Record + + buffer chan *Record + + firstRecordIndex int + lastRecordIndex int + selectedRecordIndex int + + updateType updateType + + ui *UI + filter *Filter +} + +func NewRecordsView(ui *UI, bucket *Bucket, filter *Filter) *RecordsView { + return &RecordsView{ + Box: tview.NewBox(), + bucket: bucket, + ui: ui, + filter: filter, + } +} + +func (v *RecordsView) Mount(ctx context.Context) error { + if v.onUnmount != nil { + return errors.New("try to mount already mounted component") + } + + ctx, v.onUnmount = context.WithCancel(ctx) + + tempBuffer, err := LoadRecords(ctx, v.ui.db, v.bucket.Path, v.ui.loadBufferSize) + if err != nil { + return err + } + + v.buffer = make(chan *Record, v.ui.loadBufferSize) + go func() { + defer close(v.buffer) + + for item := range tempBuffer { + if item.err != nil { + v.ui.stopOnError(err) + break + } + record := item.val + + record.Entry, _, err = v.bucket.NextParser(record.Key, record.Value) + if err != nil { + v.ui.stopOnError(err) + break + } + + if v.filter.Apply(record.Entry).Result() != common.Yes { + continue + } + + v.buffer <- record + } + }() + + return nil +} + +func (v *RecordsView) Unmount() { + if v.onUnmount == nil { + panic("try to unmount not mounted component") + } + v.onUnmount() + v.onUnmount = nil +} + +func (v *RecordsView) Update(ctx context.Context) error { + _, _, _, recordsPerPage := v.GetInnerRect() + firstRecordIndex, lastRecordIndex, selectedRecordIndex := v.getNewIndexes() + +loop: + for len(v.records) < lastRecordIndex { + select { + case <-ctx.Done(): + return nil + case record, ok := <-v.buffer: + if !ok { + break loop + } + v.records = append(v.records, record) + } + } + + // Set the update type to its default value after some specific key event + // has been handled. + v.updateType = other + + firstRecordIndex = max(0, min(firstRecordIndex, len(v.records)-recordsPerPage)) + lastRecordIndex = min(firstRecordIndex+recordsPerPage, len(v.records)) + selectedRecordIndex = min(selectedRecordIndex, lastRecordIndex-1) + + v.mu.Lock() + v.firstRecordIndex = firstRecordIndex + v.lastRecordIndex = lastRecordIndex + v.selectedRecordIndex = selectedRecordIndex + v.mu.Unlock() + + return nil +} + +func (v *RecordsView) getNewIndexes() (int, int, int) { + v.mu.RLock() + firstRecordIndex := v.firstRecordIndex + lastRecordIndex := v.lastRecordIndex + selectedRecordIndex := v.selectedRecordIndex + v.mu.RUnlock() + + _, _, _, recordsPerPage := v.GetInnerRect() + + switch v.updateType { + case moveUp: + if selectedRecordIndex != firstRecordIndex { + selectedRecordIndex-- + break + } + firstRecordIndex = max(0, firstRecordIndex-1) + lastRecordIndex = min(firstRecordIndex+recordsPerPage, len(v.records)) + selectedRecordIndex = firstRecordIndex + case moveToPrevPage: + if selectedRecordIndex != firstRecordIndex { + selectedRecordIndex = firstRecordIndex + break + } + firstRecordIndex = max(0, firstRecordIndex-recordsPerPage) + lastRecordIndex = firstRecordIndex + recordsPerPage + selectedRecordIndex = firstRecordIndex + case moveDown: + if selectedRecordIndex != lastRecordIndex-1 { + selectedRecordIndex++ + break + } + firstRecordIndex++ + lastRecordIndex++ + selectedRecordIndex++ + case moveToNextPage: + if selectedRecordIndex != lastRecordIndex-1 { + selectedRecordIndex = lastRecordIndex - 1 + break + } + firstRecordIndex += recordsPerPage + lastRecordIndex = firstRecordIndex + recordsPerPage + selectedRecordIndex = lastRecordIndex - 1 + case moveHome: + firstRecordIndex = 0 + lastRecordIndex = firstRecordIndex + recordsPerPage + selectedRecordIndex = 0 + case moveEnd: + lastRecordIndex = math.MaxInt32 + firstRecordIndex = lastRecordIndex - recordsPerPage + selectedRecordIndex = lastRecordIndex - 1 + default: + lastRecordIndex = firstRecordIndex + recordsPerPage + } + + return firstRecordIndex, lastRecordIndex, selectedRecordIndex +} + +func (v *RecordsView) GetInnerRect() (int, int, int, int) { + x, y, width, height := v.Box.GetInnerRect() + + // Left padding. + x = min(x+3, x+width-1) + width = max(width-3, 0) + + return x, y, width, height +} + +func (v *RecordsView) Draw(screen tcell.Screen) { + v.mu.RLock() + firstRecordIndex := v.firstRecordIndex + lastRecordIndex := v.lastRecordIndex + selectedRecordIndex := v.selectedRecordIndex + records := v.records + v.mu.RUnlock() + + v.DrawForSubclass(screen, v) + + x, y, width, height := v.GetInnerRect() + if height == 0 { + return + } + + // No records in that bucket. + if firstRecordIndex == lastRecordIndex { + tview.Print( + screen, "Empty Bucket", x, y, width, tview.AlignCenter, tview.Styles.PrimaryTextColor, + ) + return + } + + for index := firstRecordIndex; index < lastRecordIndex; index++ { + result := records[index].Entry + text := result.String() + + if index == selectedRecordIndex { + text = fmt.Sprintf("[:white]%s[:-]", text) + tview.Print(screen, text, x, y, width, tview.AlignLeft, tview.Styles.PrimitiveBackgroundColor) + } else { + tview.Print(screen, text, x, y, width, tview.AlignLeft, tview.Styles.PrimaryTextColor) + } + + y++ + } +} + +func (v *RecordsView) InputHandler() func(event *tcell.EventKey, _ func(p tview.Primitive)) { + return v.WrapInputHandler(func(event *tcell.EventKey, _ func(p tview.Primitive)) { + switch m, k := event.Modifiers(), event.Key(); { + case m == 0 && k == tcell.KeyPgUp: + v.updateType = moveToPrevPage + case m == 0 && k == tcell.KeyPgDn: + v.updateType = moveToNextPage + case m == 0 && k == tcell.KeyUp: + v.updateType = moveUp + case m == 0 && k == tcell.KeyDown: + v.updateType = moveDown + case m == 0 && k == tcell.KeyHome: + v.updateType = moveHome + case m == 0 && k == tcell.KeyEnd: + v.updateType = moveEnd + case k == tcell.KeyEnter: + v.mu.RLock() + selectedRecordIndex := v.selectedRecordIndex + records := v.records + v.mu.RUnlock() + if len(records) != 0 { + current := records[selectedRecordIndex] + v.ui.moveNextPage(NewDetailedView(current.Entry.DetailedString())) + } + } + }) +} diff --git a/cmd/frostfs-lens/internal/tui/types.go b/cmd/frostfs-lens/internal/tui/types.go new file mode 100644 index 000000000..4a227fe64 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/types.go @@ -0,0 +1,18 @@ +package tui + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" +) + +type Bucket struct { + Name []byte + Path [][]byte + Entry common.SchemaEntry + NextParser common.Parser +} + +type Record struct { + Key, Value []byte + Path [][]byte + Entry common.SchemaEntry +} diff --git a/cmd/frostfs-lens/internal/tui/ui.go b/cmd/frostfs-lens/internal/tui/ui.go new file mode 100644 index 000000000..701f2b331 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/ui.go @@ -0,0 +1,548 @@ +package tui + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "github.com/davecgh/go-spew/spew" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" + "go.etcd.io/bbolt" +) + +type Config struct { + LoadBufferSize int + SearchHistorySize int + LoadingIndicatorLag time.Duration +} + +var DefaultConfig = Config{ + LoadBufferSize: 100, + SearchHistorySize: 100, + LoadingIndicatorLag: 500 * time.Millisecond, +} + +type Primitive interface { + tview.Primitive + + Mount(ctx context.Context) error + Update(ctx context.Context) error + Unmount() +} + +type UI struct { + *tview.Box + + // Need to use context while updating pages those read data from a database. + // Context should be shared among all mount and updates. Current TUI library + // doesn't use contexts at all, so I do that feature by myself. + //nolint:containedctx + ctx context.Context + onStop func() + + app *tview.Application + db *bbolt.DB + + pageHistory []Primitive + mountedPage Primitive + + pageToMount Primitive + + pageStub tview.Primitive + + infoBar *tview.TextView + searchBar *InputFieldWithHistory + loadingBar *LoadingBar + helpBar *tview.TextView + + searchErrorBar *tview.TextView + + isSearching bool + isLoading atomic.Bool + isShowingError bool + isShowingHelp bool + + loadBufferSize int + + rootParser common.Parser + + loadingIndicatorLag time.Duration + + cancelLoading func() + + filters map[string]func(string) (any, error) + compositeFilters map[string]func(string) (map[string]any, error) + filterHints map[string]string +} + +func NewUI( + ctx context.Context, + app *tview.Application, + db *bbolt.DB, + rootParser common.Parser, + cfg *Config, +) *UI { + spew.Config.DisableMethods = true + + if cfg == nil { + cfg = &DefaultConfig + } + + ui := &UI{ + Box: tview.NewBox(), + + app: app, + db: db, + rootParser: rootParser, + + filters: make(map[string]func(string) (any, error)), + compositeFilters: make(map[string]func(string) (map[string]any, error)), + filterHints: make(map[string]string), + + loadBufferSize: cfg.LoadBufferSize, + loadingIndicatorLag: cfg.LoadingIndicatorLag, + } + + ui.ctx, ui.onStop = context.WithCancel(ctx) + + backgroundColor := ui.GetBackgroundColor() + textColor := tview.Styles.PrimaryTextColor + + inverseBackgroundColor := textColor + inverseTextColor := backgroundColor + + alertTextColor := tcell.ColorRed + + ui.pageStub = tview.NewBox() + + ui.infoBar = tview.NewTextView() + ui.infoBar.SetBackgroundColor(inverseBackgroundColor) + ui.infoBar.SetTextColor(inverseTextColor) + ui.infoBar.SetText( + fmt.Sprintf(" %s (press h for help, q to quit) ", db.Path()), + ) + + ui.searchBar = NewInputFieldWithHistory(cfg.SearchHistorySize) + ui.searchBar.SetFieldBackgroundColor(backgroundColor) + ui.searchBar.SetFieldTextColor(textColor) + ui.searchBar.SetLabelColor(textColor) + ui.searchBar.Focus(nil) + ui.searchBar.SetLabel("/") + + ui.searchErrorBar = tview.NewTextView() + ui.searchErrorBar.SetBackgroundColor(backgroundColor) + ui.searchErrorBar.SetTextColor(alertTextColor) + + ui.helpBar = tview.NewTextView() + ui.helpBar.SetBackgroundColor(inverseBackgroundColor) + ui.helpBar.SetTextColor(inverseTextColor) + ui.helpBar.SetText(" Press Enter for next page or Escape to exit help ") + + ui.loadingBar = NewLoadingBar(ui.triggerDraw) + + ui.pageToMount = NewBucketsView(ui, NewFilter(nil)) + + return ui +} + +func (ui *UI) checkFilterExists(typ string) bool { + if _, ok := ui.filters[typ]; ok { + return true + } + if _, ok := ui.compositeFilters[typ]; ok { + return true + } + return false +} + +func (ui *UI) AddFilter( + typ string, + parser func(string) (any, error), + helpHint string, +) error { + if ui.checkFilterExists(typ) { + return fmt.Errorf("filter %s already exists", typ) + } + ui.filters[typ] = parser + ui.filterHints[typ] = helpHint + return nil +} + +func (ui *UI) AddCompositeFilter( + typ string, + parser func(string) (map[string]any, error), + helpHint string, +) error { + if ui.checkFilterExists(typ) { + return fmt.Errorf("filter %s already exists", typ) + } + ui.compositeFilters[typ] = parser + ui.filterHints[typ] = helpHint + return nil +} + +func (ui *UI) stopOnError(err error) { + if err != nil { + ui.onStop() + ui.app.QueueEvent(tcell.NewEventError(err)) + } +} + +func (ui *UI) stop() { + ui.onStop() + ui.app.Stop() +} + +func (ui *UI) movePrevPage() { + if len(ui.pageHistory) != 0 { + ui.mountedPage.Unmount() + ui.mountedPage = ui.pageHistory[len(ui.pageHistory)-1] + ui.pageHistory = ui.pageHistory[:len(ui.pageHistory)-1] + ui.triggerDraw() + } +} + +func (ui *UI) moveNextPage(page Primitive) { + ui.pageToMount = page + ui.triggerDraw() +} + +func (ui *UI) triggerDraw() { + go ui.app.QueueUpdateDraw(func() {}) +} + +func (ui *UI) Draw(screen tcell.Screen) { + if ui.isLoading.Load() { + ui.draw(screen) + return + } + + ui.isLoading.Store(true) + + ctx, cancel := context.WithCancel(ui.ctx) + + ready := make(chan struct{}) + go func() { + ui.load(ctx) + + cancel() + close(ready) + ui.isLoading.Store(false) + }() + + select { + case <-ready: + case <-time.After(ui.loadingIndicatorLag): + ui.loadingBar.Start(ui.ctx) + ui.cancelLoading = cancel + + go func() { + <-ready + ui.loadingBar.Stop() + ui.triggerDraw() + }() + } + + ui.draw(screen) +} + +func (ui *UI) load(ctx context.Context) { + if ui.mountedPage == nil && ui.pageToMount == nil { + ui.stop() + return + } + + if ui.pageToMount != nil { + ui.mountAndUpdate(ctx) + } else { + ui.update(ctx) + } +} + +func (ui *UI) draw(screen tcell.Screen) { + ui.DrawForSubclass(screen, ui) + x, y, width, height := ui.GetInnerRect() + + var ( + pageToDraw tview.Primitive + barToDraw tview.Primitive + ) + + switch { + case ui.isShowingHelp: + pageToDraw = ui.pageStub + case ui.mountedPage != nil: + pageToDraw = ui.mountedPage + default: + pageToDraw = ui.pageStub + } + + pageToDraw.SetRect(x, y, width, height-1) + pageToDraw.Draw(screen) + + // Search bar uses cursor and we need to hide it when another bar is drawn. + screen.HideCursor() + + switch { + case ui.isLoading.Load(): + barToDraw = ui.loadingBar + case ui.isSearching: + barToDraw = ui.searchBar + case ui.isShowingError: + barToDraw = ui.searchErrorBar + case ui.isShowingHelp: + barToDraw = ui.helpBar + default: + barToDraw = ui.infoBar + } + + barToDraw.SetRect(x, y+height-1, width, 1) + barToDraw.Draw(screen) +} + +func (ui *UI) mountAndUpdate(ctx context.Context) { + defer func() { + // Operation succeeded or was canceled, either way reset page to mount. + ui.pageToMount = nil + }() + + // Mount should use app global context. + //nolint:contextcheck + err := ui.pageToMount.Mount(ui.ctx) + if err != nil { + ui.stopOnError(err) + return + } + + x, y, width, height := ui.GetInnerRect() + ui.pageToMount.SetRect(x, y, width, height-1) + + s := loadOp(ctx, ui.pageToMount.Update) + if s.err != nil { + ui.pageToMount.Unmount() + ui.stopOnError(s.err) + return + } + // Update was canceled. + if !s.done { + ui.pageToMount.Unmount() + return + } + + if ui.mountedPage != nil { + ui.pageHistory = append(ui.pageHistory, ui.mountedPage) + } + ui.mountedPage = ui.pageToMount +} + +func (ui *UI) update(ctx context.Context) { + x, y, width, height := ui.GetInnerRect() + ui.mountedPage.SetRect(x, y, width, height-1) + + s := loadOp(ctx, ui.mountedPage.Update) + if s.err != nil { + ui.stopOnError(s.err) + return + } +} + +type status struct { + done bool + err error +} + +func loadOp(ctx context.Context, op func(ctx context.Context) error) status { + errCh := make(chan error) + go func() { + errCh <- op(ctx) + }() + + select { + case <-ctx.Done(): + return status{done: false, err: nil} + case err := <-errCh: + return status{done: true, err: err} + } +} + +func (ui *UI) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) { + return ui.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) { + switch { + case ui.isLoading.Load(): + ui.handleInputOnLoading(event) + case ui.isShowingHelp: + ui.handleInputOnShowingHelp(event) + case ui.isShowingError: + ui.handleInputOnShowingError() + case ui.isSearching: + ui.handleInputOnSearching(event) + default: + ui.handleInput(event) + } + }) +} + +func (ui *UI) handleInput(event *tcell.EventKey) { + m, k, r := event.Modifiers(), event.Key(), event.Rune() + + switch { + case k == tcell.KeyEsc: + ui.movePrevPage() + case m == 0 && k == tcell.KeyRune && r == 'h': + ui.isShowingHelp = true + case m == 0 && k == tcell.KeyRune && r == '/': + ui.isSearching = true + case m == 0 && k == tcell.KeyRune && r == 'q': + ui.stop() + default: + if ui.mountedPage != nil { + ui.mountedPage.InputHandler()(event, func(tview.Primitive) {}) + } + } +} + +func (ui *UI) handleInputOnLoading(event *tcell.EventKey) { + switch k, r := event.Key(), event.Rune(); { + case k == tcell.KeyEsc: + ui.cancelLoading() + case k == tcell.KeyRune && r == 'q': + ui.stop() + } +} + +func (ui *UI) handleInputOnShowingError() { + ui.isShowingError = false + ui.isSearching = true +} + +func (ui *UI) handleInputOnShowingHelp(event *tcell.EventKey) { + k, r := event.Key(), event.Rune() + + switch { + case k == tcell.KeyEsc: + ui.isShowingHelp = false + case k == tcell.KeyRune && r == 'q': + ui.stop() + default: + } +} + +func (ui *UI) handleInputOnSearching(event *tcell.EventKey) { + m, k := event.Modifiers(), event.Key() + + switch { + case k == tcell.KeyEnter: + prompt := ui.searchBar.GetText() + + res, err := ui.processPrompt(prompt) + if err != nil { + ui.isShowingError = true + ui.isSearching = false + ui.searchErrorBar.SetText(err.Error() + " (press any key to continue)") + return + } + + switch ui.mountedPage.(type) { + case *BucketsView: + ui.moveNextPage(NewBucketsView(ui, res)) + case *RecordsView: + bucket := ui.mountedPage.(*RecordsView).bucket + ui.moveNextPage(NewRecordsView(ui, bucket, res)) + } + + if ui.searchBar.GetText() != "" { + ui.searchBar.AddToHistory(ui.searchBar.GetText()) + } + + ui.searchBar.SetText("") + ui.isSearching = false + case k == tcell.KeyEsc: + ui.isSearching = false + case (k == tcell.KeyBackspace2 || m&tcell.ModCtrl != 0 && k == tcell.KeyETB) && len(ui.searchBar.GetText()) == 0: + ui.isSearching = false + default: + ui.searchBar.InputHandler()(event, func(tview.Primitive) {}) + } + + ui.Box.MouseHandler() +} + +func (ui *UI) WithPrompt(prompt string) error { + filter, err := ui.processPrompt(prompt) + if err != nil { + return err + } + + ui.pageToMount = NewBucketsView(ui, filter) + + if prompt != "" { + ui.searchBar.AddToHistory(prompt) + } + + return nil +} + +func (ui *UI) processPrompt(prompt string) (filter *Filter, err error) { + if prompt == "" { + return NewFilter(nil), nil + } + + filterMap := make(map[string]any) + + for _, filterString := range strings.Split(prompt, "+") { + parts := strings.Split(filterString, ":") + if len(parts) != 2 { + return nil, errors.New("expected 'tag:value [+ tag:value]...'") + } + + filterTag := strings.TrimSpace(parts[0]) + filterValueString := strings.TrimSpace(parts[1]) + + if _, exists := filterMap[filterTag]; exists { + return nil, fmt.Errorf("duplicate filter tag '%s'", filterTag) + } + + parser, ok := ui.filters[filterTag] + if ok { + filterValue, err := parser(filterValueString) + if err != nil { + return nil, fmt.Errorf("can't parse '%s' filter value: %w", filterTag, err) + } + + filterMap[filterTag] = filterValue + continue + } + + compositeParser, ok := ui.compositeFilters[filterTag] + if ok { + compositeFilterValue, err := compositeParser(filterValueString) + if err != nil { + return nil, fmt.Errorf( + "can't parse '%s' filter value '%s': %w", + filterTag, filterValueString, err, + ) + } + + for tag, value := range compositeFilterValue { + if _, exists := filterMap[tag]; exists { + return nil, fmt.Errorf( + "found duplicate filter tag '%s' while processing composite filter with tag '%s'", + tag, filterTag, + ) + } + + filterMap[tag] = value + } + continue + } + + return nil, fmt.Errorf("unknown filter tag '%s'", filterTag) + } + + return NewFilter(filterMap), nil +} diff --git a/cmd/frostfs-lens/internal/tui/util.go b/cmd/frostfs-lens/internal/tui/util.go new file mode 100644 index 000000000..d4e13b2a9 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/util.go @@ -0,0 +1,97 @@ +package tui + +import ( + "errors" + "strings" + + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/mr-tron/base58" +) + +func CIDParser(s string) (any, error) { + data, err := base58.Decode(s) + if err != nil { + return nil, err + } + var id cid.ID + if err = id.Decode(data); err != nil { + return nil, err + } + return id, nil +} + +func OIDParser(s string) (any, error) { + data, err := base58.Decode(s) + if err != nil { + return nil, err + } + var id oid.ID + if err = id.Decode(data); err != nil { + return nil, err + } + return id, nil +} + +func AddressParser(s string) (map[string]any, error) { + m := make(map[string]any) + + parts := strings.Split(s, "/") + if len(parts) != 2 { + return nil, errors.New("expected /") + } + cnr, err := CIDParser(parts[0]) + if err != nil { + return nil, err + } + obj, err := OIDParser(parts[1]) + if err != nil { + return nil, err + } + + m["cid"] = cnr + m["oid"] = obj + + return m, nil +} + +func keyParser(s string) (any, error) { + if s == "" { + return nil, errors.New("empty attribute key") + } + return s, nil +} + +func valueParser(s string) (any, error) { + if s == "" { + return nil, errors.New("empty attribute value") + } + return s, nil +} + +func AttributeParser(s string) (map[string]any, error) { + m := make(map[string]any) + + parts := strings.Split(s, "/") + if len(parts) != 1 && len(parts) != 2 { + return nil, errors.New("expected or /") + } + + key, err := keyParser(parts[0]) + if err != nil { + return nil, err + } + m["key"] = key + + if len(parts) == 1 { + return m, nil + } + + value, err := valueParser(parts[1]) + if err != nil { + return nil, err + } + m["value"] = value + + return m, nil +} diff --git a/go.mod b/go.mod index 927d978ac..4e8f70fce 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 github.com/prometheus/client_golang v1.19.0 + github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130 github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 @@ -106,7 +107,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.11.0 // indirect diff --git a/go.sum b/go.sum index 3fb24b3903f35783a6d27f2f7175b43100ea7a6c..858405f8ddc26b34ef0163fb73c250026c90a2a2 100644 GIT binary patch delta 278 zcmZ3qhw1lTrVTrT>r2WqQ_B^~4D^6d*T}%g#K6qR)X>7z+{8#X$sp0($kI5;!qC`2 zA;Zur-_SX-B;UwA{rjz$Z&PIKLvaB-F(%F)+QvFf-DYD?PI$ zqclk`IX_pwD6=eIAG<{!m^SIB=j-L>r+_RpFv@iEtTOl3ceKoMcP=Z+tnvvc%P%Z4 zNp|(DFfTMU3Gy%vEKSKSDlVQJ7$P(IaPS%fbCAaH(3G<5aEnscjH1XiA7_)Os+0mx kqpFgM07Fkx3nOh4OaEk3-$FB!ut11`Af?d7$UHsE*TOfx%EVGH8$>!;yDV_kDt{iIs -- 2.45.2 From 4e61f42620a5003188b72c70b179dfca02e25714 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 16 Aug 2024 17:27:35 +0300 Subject: [PATCH 5/7] [#1223] lens/tui: Add app help Signed-off-by: Aleksey Savchuk --- .../internal/tui/help-pages/hotkeys.txt | 38 +++++++ .../internal/tui/help-pages/searching.txt | 26 +++++ cmd/frostfs-lens/internal/tui/help.go | 101 ++++++++++++++++++ cmd/frostfs-lens/internal/tui/ui.go | 15 ++- 4 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-lens/internal/tui/help-pages/hotkeys.txt create mode 100644 cmd/frostfs-lens/internal/tui/help-pages/searching.txt create mode 100644 cmd/frostfs-lens/internal/tui/help.go diff --git a/cmd/frostfs-lens/internal/tui/help-pages/hotkeys.txt b/cmd/frostfs-lens/internal/tui/help-pages/hotkeys.txt new file mode 100644 index 000000000..c371b34e9 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/help-pages/hotkeys.txt @@ -0,0 +1,38 @@ +[green::b]HOTKEYS[-::-] + + [green::b]Navigation[-::-] + + [yellow::b]Down Arrow[-::-] / [yellow::b]j[-::-] + Scroll down. + + [yellow::b]Up Arrow[-::-] / [yellow::b]k[-::-] + Scroll up. + + [yellow::b]Page Down[-::-] / [yellow::b]Ctrl-f[-::-] + Scroll down by a full page. + + [yellow::b]Page Up[-::-] / [yellow::b]Ctrl-b[-::-] + Scroll up by a full page. + + [green::b]Actions[-::-] + + [yellow::b]Enter[-::-] + Perform actions based on the current context: + - In Buckets View: + - Expand/collapse the selected bucket to show/hide its nested buckets. + - If no nested buckets exist, navigate to the selected bucket's records. + - In Records View: Open the detailed view of the selected record. + + [yellow::b]Escape[-::-] + Return to the previous page, opposite of [yellow::b]Enter[-::-]. + + Refer to the [green::b]SEARCHING[-::-] section for more specific actions. + + + [green::b]Alternative Action Hotkeys[-::-] + + [yellow::b]Ctrl-r[-::-] + Directly navigate to the selected bucket's records. + + [yellow::b]Ctrl-d[-::-] + Access the detailed view of the selected bucket. diff --git a/cmd/frostfs-lens/internal/tui/help-pages/searching.txt b/cmd/frostfs-lens/internal/tui/help-pages/searching.txt new file mode 100644 index 000000000..bc2be512b --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/help-pages/searching.txt @@ -0,0 +1,26 @@ +[green::b]SEARCHING[-::-] + + [green::b]Hotkeys[-::-] + + [yellow::b]/[-::-] + Initiate the search prompt. + - The prompt follows this syntax: [yellow::b]tag:value [+ tag:value]...[-::-] + - Multiple filter can be combined with [yellow::b]+[-::-], the result is an intersection of those filters' result sets. + - Any leading and trailing whitespace will be ignored. + - An empty prompt will return all results with no filters applied. + - Refer to the [green::b]Available Search Filters[-::-] section below for a list of valid filter tags. + + [yellow::b]Enter[-::-] + Execute the search based on the entered prompt. + - If the prompt is invalid, an error message will be displayed. + + [yellow::b]Escape[-::-] + Exit the search prompt without performing a search. + + [yellow::b]Down Arrow[-::-], [yellow::b]Up Arrow[-::-] + Scroll through the search history. + + + [green::b]Available Search Filters[-::-] + +%s diff --git a/cmd/frostfs-lens/internal/tui/help.go b/cmd/frostfs-lens/internal/tui/help.go new file mode 100644 index 000000000..3ab8fede0 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/help.go @@ -0,0 +1,101 @@ +package tui + +import ( + _ "embed" + "fmt" + "strings" + + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +var ( + //go:embed help-pages/hotkeys.txt + hotkeysHelpText string + + //go:embed help-pages/searching.txt + searchingHelpText string +) + +type HelpPage struct { + *tview.Box + pages []*tview.TextView + currentPage int + + filters []string + filterHints map[string]string +} + +func NewHelpPage(filters []string, hints map[string]string) *HelpPage { + hp := &HelpPage{ + Box: tview.NewBox(), + filters: filters, + filterHints: hints, + } + + page := tview.NewTextView(). + SetDynamicColors(true). + SetText(hotkeysHelpText) + hp.addPage(page) + + page = tview.NewTextView(). + SetDynamicColors(true). + SetText(fmt.Sprintf(searchingHelpText, hp.getFiltersText())) + hp.addPage(page) + + return hp +} + +func (hp *HelpPage) addPage(page *tview.TextView) { + hp.pages = append(hp.pages, page) +} + +func (hp *HelpPage) getFiltersText() string { + if len(hp.filters) == 0 { + return "\t\tNo filters defined.\n" + } + + filtersText := strings.Builder{} + gapSize := 4 + + tagMaxWidth := 3 + for _, filter := range hp.filters { + tagMaxWidth = max(tagMaxWidth, len(filter)) + } + filtersText.WriteString("\t\t[yellow::b]Tag") + filtersText.WriteString(strings.Repeat(" ", gapSize)) + filtersText.WriteString("\tValue[-::-]\n\n") + + for _, filter := range hp.filters { + filtersText.WriteString("\t\t") + filtersText.WriteString(filter) + filtersText.WriteString(strings.Repeat(" ", tagMaxWidth-len(filter)+gapSize)) + filtersText.WriteString(hp.filterHints[filter]) + filtersText.WriteRune('\n') + } + + return filtersText.String() +} + +func (hp *HelpPage) Draw(screen tcell.Screen) { + x, y, width, height := hp.GetInnerRect() + hp.pages[hp.currentPage].SetRect(x+1, y+1, width-2, height-2) + hp.pages[hp.currentPage].Draw(screen) +} + +func (hp *HelpPage) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) { + return hp.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) { + if event.Key() == tcell.KeyEnter { + hp.currentPage++ + hp.currentPage %= len(hp.pages) + return + } + hp.pages[hp.currentPage].InputHandler()(event, func(tview.Primitive) {}) + }) +} + +func (hp *HelpPage) MouseHandler() func(action tview.MouseAction, event *tcell.EventMouse, setFocus func(p tview.Primitive)) (consumed bool, capture tview.Primitive) { + return hp.WrapMouseHandler(func(action tview.MouseAction, event *tcell.EventMouse, _ func(tview.Primitive)) (consumed bool, capture tview.Primitive) { + return hp.pages[hp.currentPage].MouseHandler()(action, event, func(tview.Primitive) {}) + }) +} diff --git a/cmd/frostfs-lens/internal/tui/ui.go b/cmd/frostfs-lens/internal/tui/ui.go index 701f2b331..bcc082821 100644 --- a/cmd/frostfs-lens/internal/tui/ui.go +++ b/cmd/frostfs-lens/internal/tui/ui.go @@ -60,6 +60,8 @@ type UI struct { loadingBar *LoadingBar helpBar *tview.TextView + helpPage *HelpPage + searchErrorBar *tview.TextView isSearching bool @@ -275,7 +277,17 @@ func (ui *UI) draw(screen tcell.Screen) { switch { case ui.isShowingHelp: - pageToDraw = ui.pageStub + if ui.helpPage == nil { + var filters []string + for f := range ui.filters { + filters = append(filters, f) + } + for f := range ui.compositeFilters { + filters = append(filters, f) + } + ui.helpPage = NewHelpPage(filters, ui.filterHints) + } + pageToDraw = ui.helpPage case ui.mountedPage != nil: pageToDraw = ui.mountedPage default: @@ -429,6 +441,7 @@ func (ui *UI) handleInputOnShowingHelp(event *tcell.EventKey) { case k == tcell.KeyRune && r == 'q': ui.stop() default: + ui.helpPage.InputHandler()(event, func(tview.Primitive) {}) } } -- 2.45.2 From fd5cc92eb5f5466009e64d2b523cf4bb30f8a8ac Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 19 Aug 2024 18:02:11 +0300 Subject: [PATCH 6/7] [#1223] lens/tui: Add TUI app for write cache Signed-off-by: Aleksey Savchuk --- cmd/frostfs-lens/internal/writecache/root.go | 2 +- cmd/frostfs-lens/internal/writecache/tui.go | 79 ++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-lens/internal/writecache/tui.go diff --git a/cmd/frostfs-lens/internal/writecache/root.go b/cmd/frostfs-lens/internal/writecache/root.go index eb3b325b6..d7d6db240 100644 --- a/cmd/frostfs-lens/internal/writecache/root.go +++ b/cmd/frostfs-lens/internal/writecache/root.go @@ -17,5 +17,5 @@ var Root = &cobra.Command{ } func init() { - Root.AddCommand(listCMD, inspectCMD) + Root.AddCommand(listCMD, inspectCMD, tuiCMD) } diff --git a/cmd/frostfs-lens/internal/writecache/tui.go b/cmd/frostfs-lens/internal/writecache/tui.go new file mode 100644 index 000000000..6b7532b08 --- /dev/null +++ b/cmd/frostfs-lens/internal/writecache/tui.go @@ -0,0 +1,79 @@ +package writecache + +import ( + "context" + "fmt" + + common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" + schema "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui" + "github.com/rivo/tview" + "github.com/spf13/cobra" + "go.etcd.io/bbolt" +) + +var tuiCMD = &cobra.Command{ + Use: "explore", + Short: "Write cache exploration with a terminal UI", + Long: `Launch a terminal UI to explore write cache and search for data. + +Available search filters: +- cid CID +- oid OID +- addr CID/OID +`, + Run: tuiFunc, +} + +var initialPrompt string + +func init() { + common.AddComponentPathFlag(tuiCMD, &vPath) + + tuiCMD.Flags().StringVar( + &initialPrompt, + "filter", + "", + "Filter prompt to start with, format 'tag:value [+ tag:value]...'", + ) +} + +func tuiFunc(cmd *cobra.Command, _ []string) { + common.ExitOnErr(cmd, runTUI(cmd)) +} + +func runTUI(cmd *cobra.Command) error { + db, err := openDB(false) + if err != nil { + return fmt.Errorf("couldn't open database: %w", err) + } + defer db.Close() + + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() + + app := tview.NewApplication() + ui := tui.NewUI(ctx, app, db, schema.WritecacheParser, nil) + + _ = ui.AddFilter("cid", tui.CIDParser, "CID") + _ = ui.AddFilter("oid", tui.OIDParser, "OID") + _ = ui.AddCompositeFilter("addr", tui.AddressParser, "CID/OID") + + err = ui.WithPrompt(initialPrompt) + if err != nil { + return fmt.Errorf("invalid filter prompt: %w", err) + } + + app.SetRoot(ui, true).SetFocus(ui) + return app.Run() +} + +func openDB(writable bool) (*bbolt.DB, error) { + db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{ + ReadOnly: !writable, + }) + if err != nil { + return nil, err + } + return db, nil +} -- 2.45.2 From 37a903d27a9c6a06049c17f5388324b4167cefb0 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 22 Aug 2024 15:07:51 +0300 Subject: [PATCH 7/7] [#1223] lens/tui: Add TUI app for blobovnicza Signed-off-by: Aleksey Savchuk --- cmd/frostfs-lens/internal/blobovnicza/root.go | 2 +- cmd/frostfs-lens/internal/blobovnicza/tui.go | 79 ++++++++++++++ .../internal/schema/blobovnicza/parsers.go | 96 +++++++++++++++++ .../internal/schema/blobovnicza/types.go | 101 ++++++++++++++++++ 4 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-lens/internal/blobovnicza/tui.go create mode 100644 cmd/frostfs-lens/internal/schema/blobovnicza/parsers.go create mode 100644 cmd/frostfs-lens/internal/schema/blobovnicza/types.go diff --git a/cmd/frostfs-lens/internal/blobovnicza/root.go b/cmd/frostfs-lens/internal/blobovnicza/root.go index 0a0cd955d..9d8ef3dad 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/root.go +++ b/cmd/frostfs-lens/internal/blobovnicza/root.go @@ -19,7 +19,7 @@ var Root = &cobra.Command{ } func init() { - Root.AddCommand(listCMD, inspectCMD) + Root.AddCommand(listCMD, inspectCMD, tuiCMD) } func openBlobovnicza(cmd *cobra.Command) *blobovnicza.Blobovnicza { diff --git a/cmd/frostfs-lens/internal/blobovnicza/tui.go b/cmd/frostfs-lens/internal/blobovnicza/tui.go new file mode 100644 index 000000000..eb4a5ff59 --- /dev/null +++ b/cmd/frostfs-lens/internal/blobovnicza/tui.go @@ -0,0 +1,79 @@ +package blobovnicza + +import ( + "context" + "fmt" + + common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" + schema "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui" + "github.com/rivo/tview" + "github.com/spf13/cobra" + "go.etcd.io/bbolt" +) + +var tuiCMD = &cobra.Command{ + Use: "explore", + Short: "Blobovnicza exploration with a terminal UI", + Long: `Launch a terminal UI to explore blobovnicza and search for data. + +Available search filters: +- cid CID +- oid OID +- addr CID/OID +`, + Run: tuiFunc, +} + +var initialPrompt string + +func init() { + common.AddComponentPathFlag(tuiCMD, &vPath) + + tuiCMD.Flags().StringVar( + &initialPrompt, + "filter", + "", + "Filter prompt to start with, format 'tag:value [+ tag:value]...'", + ) +} + +func tuiFunc(cmd *cobra.Command, _ []string) { + common.ExitOnErr(cmd, runTUI(cmd)) +} + +func runTUI(cmd *cobra.Command) error { + db, err := openDB(false) + if err != nil { + return fmt.Errorf("couldn't open database: %w", err) + } + defer db.Close() + + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() + + app := tview.NewApplication() + ui := tui.NewUI(ctx, app, db, schema.BlobovniczaParser, nil) + + _ = ui.AddFilter("cid", tui.CIDParser, "CID") + _ = ui.AddFilter("oid", tui.OIDParser, "OID") + _ = ui.AddCompositeFilter("addr", tui.AddressParser, "CID/OID") + + err = ui.WithPrompt(initialPrompt) + if err != nil { + return fmt.Errorf("invalid filter prompt: %w", err) + } + + app.SetRoot(ui, true).SetFocus(ui) + return app.Run() +} + +func openDB(writable bool) (*bbolt.DB, error) { + db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{ + ReadOnly: !writable, + }) + if err != nil { + return nil, err + } + return db, nil +} diff --git a/cmd/frostfs-lens/internal/schema/blobovnicza/parsers.go b/cmd/frostfs-lens/internal/schema/blobovnicza/parsers.go new file mode 100644 index 000000000..02b6cf414 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/blobovnicza/parsers.go @@ -0,0 +1,96 @@ +package blobovnicza + +import ( + "encoding/binary" + "errors" + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/mr-tron/base58" +) + +var BlobovniczaParser = common.WithFallback( + common.Any( + MetaBucketParser, + BucketParser, + ), + common.RawParser.ToFallbackParser(), +) + +func MetaBucketParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, errors.New("not a bucket") + } + + if string(key) != "META" { + return nil, nil, errors.New("invalid bucket name") + } + + return &MetaBucket{}, MetaRecordParser, nil +} + +func MetaRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var r MetaRecord + + if len(key) == 0 { + return nil, nil, errors.New("invalid key") + } + + r.label = string(key) + r.count = binary.LittleEndian.Uint64(value) + + return &r, nil, nil +} + +func BucketParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, errors.New("not a bucket") + } + + size, n := binary.Varint(key) + if n <= 0 { + return nil, nil, errors.New("invalid size") + } + + return &Bucket{size: size}, RecordParser, nil +} + +func RecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + parts := strings.Split(string(key), "/") + + if len(parts) != 2 { + return nil, nil, errors.New("invalid key, expected address string /") + } + + cnrRaw, err := base58.Decode(parts[0]) + if err != nil { + return nil, nil, errors.New("can't decode CID string") + } + objRaw, err := base58.Decode(parts[1]) + if err != nil { + return nil, nil, errors.New("can't decode OID string") + } + + cnr := cid.ID{} + if err := cnr.Decode(cnrRaw); err != nil { + return nil, nil, fmt.Errorf("can't decode CID: %w", err) + } + obj := oid.ID{} + if err := obj.Decode(objRaw); err != nil { + return nil, nil, fmt.Errorf("can't decode OID: %w", err) + } + + var r Record + + r.addr.SetContainer(cnr) + r.addr.SetObject(obj) + + if err := r.object.Unmarshal(value); err != nil { + return nil, nil, errors.New("can't unmarshal object") + } + + return &r, nil, nil +} diff --git a/cmd/frostfs-lens/internal/schema/blobovnicza/types.go b/cmd/frostfs-lens/internal/schema/blobovnicza/types.go new file mode 100644 index 000000000..c7ed08cdd --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/blobovnicza/types.go @@ -0,0 +1,101 @@ +package blobovnicza + +import ( + "fmt" + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/davecgh/go-spew/spew" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type ( + MetaBucket struct{} + + MetaRecord struct { + label string + count uint64 + } + + Bucket struct { + size int64 + } + + Record struct { + addr oid.Address + object objectSDK.Object + } +) + +func (b *MetaBucket) String() string { + return common.FormatSimple("META", tcell.ColorLime) +} + +func (b *MetaBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *MetaBucket) Filter(string, any) common.FilterResult { + return common.No +} + +func (r *MetaRecord) String() string { + return fmt.Sprintf("%-11s %c %d", r.label, tview.Borders.Vertical, r.count) +} + +func (r *MetaRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *MetaRecord) Filter(string, any) common.FilterResult { + return common.No +} + +func (b *Bucket) String() string { + return common.FormatSimple(strconv.FormatInt(b.size, 10), tcell.ColorLime) +} + +func (b *Bucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *Bucket) Filter(typ string, _ any) common.FilterResult { + switch typ { + case "cid": + return common.Maybe + case "oid": + return common.Maybe + default: + return common.No + } +} + +func (r *Record) String() string { + return fmt.Sprintf( + "CID %s OID %s %c Object {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Container()), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Object()), tcell.ColorAqua), + tview.Borders.Vertical, + ) +} + +func (r *Record) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *Record) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.addr.Container().Equals(id), common.Yes, common.No) + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.addr.Object().Equals(id), common.Yes, common.No) + default: + return common.No + } +} -- 2.45.2