lens: Add ability to view raw data in metabase #1246

Merged
fyrchik merged 7 commits from a-savchuk/frostfs-node:feat/1223-lens-meta-tui into master 2024-09-05 08:03:53 +00:00
44 changed files with 4243 additions and 4 deletions

View file

@ -19,7 +19,7 @@ var Root = &cobra.Command{
}
func init() {
Root.AddCommand(listCMD, inspectCMD)
Root.AddCommand(listCMD, inspectCMD, tuiCMD)
}
func openBlobovnicza(cmd *cobra.Command) *blobovnicza.Blobovnicza {

View file

@ -0,0 +1,79 @@
package blobovnicza
import (
"context"
"fmt"
common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal"
schema "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/blobovnicza"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui"
"github.com/rivo/tview"
"github.com/spf13/cobra"
"go.etcd.io/bbolt"
)
var tuiCMD = &cobra.Command{
Use: "explore",
aarifullin marked this conversation as resolved Outdated

What does TUI stand for? Can you leave a comment, please

What does `TUI` stand for? Can you leave a comment, please

I've changed the command name based on that comment. Now it looks like this

$ frostfs-lens meta explore --path=meta.db

Also I've made the command help message more clear.

I've changed the command name based on [that comment](https://git.frostfs.info/TrueCloudLab/frostfs-node/pulls/1246#issuecomment-48252). Now it looks like this ```bash $ frostfs-lens meta explore --path=meta.db ``` Also I've made the command help message more clear.
Short: "Blobovnicza exploration with a terminal UI",
Long: `Launch a terminal UI to explore blobovnicza and search for data.
Available search filters:
- cid CID
- oid OID
- addr CID/OID
`,
Run: tuiFunc,
}
var initialPrompt string
func init() {
common.AddComponentPathFlag(tuiCMD, &vPath)
tuiCMD.Flags().StringVar(
&initialPrompt,
"filter",
"",
"Filter prompt to start with, format 'tag:value [+ tag:value]...'",
)
}
func tuiFunc(cmd *cobra.Command, _ []string) {
common.ExitOnErr(cmd, runTUI(cmd))
aarifullin marked this conversation as resolved Outdated

You pass cmd to runTUI. So, you can declare runTUI without returning error.
You have already exit with common.ExitOnErr(cmd, err) when openDB is failed. Let's make it for all errors

func runTUI(cmd *cobra.Command) {
   /*...*/
   common.ExitOnErr("prompt error: %w", ui.WithPrompt(initialPrompt))
   /*...*/
   common.ExitOnErr("application run error: %w", app.Run())
}
You pass `cmd` to `runTUI`. So, you can declare `runTUI` without returning `error`. You have already exit with `common.ExitOnErr(cmd, err)` when `openDB` is failed. Let's make it for all errors ```go func runTUI(cmd *cobra.Command) { /*...*/ common.ExitOnErr("prompt error: %w", ui.WithPrompt(initialPrompt)) /*...*/ common.ExitOnErr("application run error: %w", app.Run()) } ```

That makes sense especially your comment about error handling after opening a database. However, I have several defers need to be called, so I think it'd be easier to handle them in runTUI and then exit with an error in tuiFunc. Please feel free to correct me if I'm wrong.

I've changed error handling after opening a database, it now returns an error to the calling function.

That makes sense especially your comment about error handling after opening a database. However, I have several `defer`s need to be called, so I think it'd be easier to handle them in `runTUI` and then exit with an error in `tuiFunc`. Please feel free to correct me if I'm wrong. I've changed error handling after opening a database, it now returns an error to the calling function.
}
func runTUI(cmd *cobra.Command) error {
db, err := openDB(false)
if err != nil {
return fmt.Errorf("couldn't open database: %w", err)
}
defer db.Close()
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
app := tview.NewApplication()
ui := tui.NewUI(ctx, app, db, schema.BlobovniczaParser, nil)
_ = ui.AddFilter("cid", tui.CIDParser, "CID")
_ = ui.AddFilter("oid", tui.OIDParser, "OID")
_ = ui.AddCompositeFilter("addr", tui.AddressParser, "CID/OID")
err = ui.WithPrompt(initialPrompt)
if err != nil {
return fmt.Errorf("invalid filter prompt: %w", err)
}
app.SetRoot(ui, true).SetFocus(ui)
return app.Run()
}
func openDB(writable bool) (*bbolt.DB, error) {
db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{
ReadOnly: !writable,
})
if err != nil {
return nil, err
}
return db, nil
}

View file

@ -32,6 +32,7 @@ func init() {
inspectCMD,
listGraveyardCMD,
listGarbageCMD,
tuiCMD,
)
}

View file

@ -0,0 +1,82 @@
package meta
import (
"context"
"fmt"
common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal"
schema "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui"
"github.com/rivo/tview"
"github.com/spf13/cobra"
"go.etcd.io/bbolt"
)
var tuiCMD = &cobra.Command{
Use: "explore",

TUI is a tool name. For command it is better something like metabase explore IMHO.

`TUI` is a tool name. For command it is better something like `metabase explore` IMHO.

Changed the command name.

Changed the command name.
Short: "Metabase exploration with a terminal UI",
Long: `Launch a terminal UI to explore metabase and search for data.
Available search filters:
- cid CID
- oid OID
- addr CID/OID
- attr key[/value]
`,
Run: tuiFunc,
}
var initialPrompt string
func init() {
common.AddComponentPathFlag(tuiCMD, &vPath)
tuiCMD.Flags().StringVar(
fyrchik marked this conversation as resolved Outdated

The command could be run on production databases. It would be nice to have an additional level of security.
I suggest always opening the DB in read-only mode. If we need, we will add --read-write flag later.

The command could be run on production databases. It would be nice to have an additional level of security. I suggest always opening the DB in read-only mode. If we need, we will add `--read-write` flag later.

I added an option to openBoltDB function and made a database open in read-only mode only for now.

I added an option to `openBoltDB` function and made a database open in read-only mode only for now.
&initialPrompt,
"filter",
"",
"Filter prompt to start with, format 'tag:value [+ tag:value]...'",
)
}
func tuiFunc(cmd *cobra.Command, _ []string) {
common.ExitOnErr(cmd, runTUI(cmd))
}
func runTUI(cmd *cobra.Command) error {
db, err := openDB(false)
if err != nil {
return fmt.Errorf("couldn't open database: %w", err)

Please remove commented code.

Please remove commented code.

Removed.

Removed.
}
defer db.Close()
// Need if app was stopped with Ctrl-C.
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
app := tview.NewApplication()
ui := tui.NewUI(ctx, app, db, schema.MetabaseParser, nil)
_ = ui.AddFilter("cid", tui.CIDParser, "CID")
_ = ui.AddFilter("oid", tui.OIDParser, "OID")
_ = ui.AddCompositeFilter("addr", tui.AddressParser, "CID/OID")
_ = ui.AddCompositeFilter("attr", tui.AttributeParser, "key[/value]")
err = ui.WithPrompt(initialPrompt)
if err != nil {
return fmt.Errorf("invalid filter prompt: %w", err)
}
app.SetRoot(ui, true).SetFocus(ui)
return app.Run()
}
func openDB(writable bool) (*bbolt.DB, error) {
db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{
ReadOnly: !writable,
})
if err != nil {
return nil, err
}
return db, nil
}

View file

@ -0,0 +1,96 @@
package blobovnicza
import (
"encoding/binary"
"errors"
"fmt"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/mr-tron/base58"
)
var BlobovniczaParser = common.WithFallback(
common.Any(
MetaBucketParser,
BucketParser,
),
common.RawParser.ToFallbackParser(),
)
func MetaBucketParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if value != nil {
return nil, nil, errors.New("not a bucket")
}
if string(key) != "META" {
return nil, nil, errors.New("invalid bucket name")
}
return &MetaBucket{}, MetaRecordParser, nil
}
func MetaRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
var r MetaRecord
if len(key) == 0 {
return nil, nil, errors.New("invalid key")
}
r.label = string(key)
r.count = binary.LittleEndian.Uint64(value)
return &r, nil, nil
}
func BucketParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if value != nil {
return nil, nil, errors.New("not a bucket")
}
size, n := binary.Varint(key)
if n <= 0 {
return nil, nil, errors.New("invalid size")
}
return &Bucket{size: size}, RecordParser, nil
}
func RecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 2 {
return nil, nil, errors.New("invalid key, expected address string <CID>/<OID>")
}
cnrRaw, err := base58.Decode(parts[0])
if err != nil {
return nil, nil, errors.New("can't decode CID string")
}
objRaw, err := base58.Decode(parts[1])
if err != nil {
return nil, nil, errors.New("can't decode OID string")
}
cnr := cid.ID{}
if err := cnr.Decode(cnrRaw); err != nil {
return nil, nil, fmt.Errorf("can't decode CID: %w", err)
}
obj := oid.ID{}
if err := obj.Decode(objRaw); err != nil {
return nil, nil, fmt.Errorf("can't decode OID: %w", err)
}
var r Record
r.addr.SetContainer(cnr)
r.addr.SetObject(obj)
if err := r.object.Unmarshal(value); err != nil {
return nil, nil, errors.New("can't unmarshal object")
}
return &r, nil, nil
}

View file

@ -0,0 +1,101 @@
package blobovnicza
import (
"fmt"
"strconv"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/davecgh/go-spew/spew"
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
)
type (
MetaBucket struct{}
MetaRecord struct {
label string
count uint64
}
Bucket struct {
size int64
}
Record struct {
addr oid.Address
object objectSDK.Object
}
)
func (b *MetaBucket) String() string {
return common.FormatSimple("META", tcell.ColorLime)
}
func (b *MetaBucket) DetailedString() string {
return spew.Sdump(*b)
}
func (b *MetaBucket) Filter(string, any) common.FilterResult {
return common.No
}
func (r *MetaRecord) String() string {
return fmt.Sprintf("%-11s %c %d", r.label, tview.Borders.Vertical, r.count)
}
func (r *MetaRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *MetaRecord) Filter(string, any) common.FilterResult {
return common.No
}
func (b *Bucket) String() string {
return common.FormatSimple(strconv.FormatInt(b.size, 10), tcell.ColorLime)
}
func (b *Bucket) DetailedString() string {
return spew.Sdump(*b)
}
func (b *Bucket) Filter(typ string, _ any) common.FilterResult {
switch typ {
case "cid":
return common.Maybe
case "oid":
return common.Maybe
default:
return common.No
}
}
func (r *Record) String() string {
return fmt.Sprintf(
"CID %s OID %s %c Object {...}",
common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Container()), tcell.ColorAqua),
common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Object()), tcell.ColorAqua),
tview.Borders.Vertical,
)
}
func (r *Record) DetailedString() string {
return spew.Sdump(*r)
}
func (r *Record) Filter(typ string, val any) common.FilterResult {
switch typ {
case "cid":
id := val.(cid.ID)
return common.IfThenElse(r.addr.Container().Equals(id), common.Yes, common.No)
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.addr.Object().Equals(id), common.Yes, common.No)
default:
return common.No
}
}

View file

@ -0,0 +1,43 @@
package common
import (
"fmt"
"strconv"
"github.com/gdamore/tcell/v2"
)
type FormatOptions struct {
Color tcell.Color
Bold,
Italic,
Underline,
StrikeThrough bool
}
func Format(s string, opts FormatOptions) string {
var boldTag, italicTag, underlineTag, strikeThroughTag string
switch {
case opts.Bold:
boldTag = "b"
case opts.Italic:
italicTag = "i"
case opts.Underline:
underlineTag = "u"
case opts.StrikeThrough:
strikeThroughTag = "s"
}
attrs := fmt.Sprintf(
"%s%s%s%s", boldTag, italicTag, underlineTag, strikeThroughTag,
)
color := strconv.FormatInt(int64(opts.Color.Hex()), 16)
return fmt.Sprintf("[#%06s::%s]%s[-::-]", color, attrs, s)
}
func FormatSimple(s string, c tcell.Color) string {
return Format(s, FormatOptions{Color: c})
}

View file

@ -0,0 +1,29 @@
package common
import (
"github.com/davecgh/go-spew/spew"
"github.com/gdamore/tcell/v2"
"github.com/mr-tron/base58"
)
type RawEntry struct {
key, value []byte
}
var RawParser Parser = rawParser
func rawParser(key, value []byte) (SchemaEntry, Parser, error) {
return &RawEntry{key: key, value: value}, rawParser, nil
}
func (r *RawEntry) String() string {
return FormatSimple(base58.Encode(r.key), tcell.ColorRed)
}
func (r *RawEntry) DetailedString() string {
return spew.Sdump(r)
}
func (r *RawEntry) Filter(string, any) FilterResult {
return No
}

View file

@ -0,0 +1,81 @@
package common
import (
"errors"
"fmt"
)
type FilterResult byte
const (
No FilterResult = iota
Maybe
Yes
)
func IfThenElse(condition bool, onSuccess, onFailure FilterResult) FilterResult {
var res FilterResult
if condition {
res = onSuccess
} else {
res = onFailure
}
return res
}
type SchemaEntry interface {
String() string
DetailedString() string
Filter(typ string, val any) FilterResult
}
type (
Parser func(key, value []byte) (SchemaEntry, Parser, error)
FallbackParser func(key, value []byte) (SchemaEntry, Parser)
)
func Any(parsers ...Parser) Parser {
return func(key, value []byte) (SchemaEntry, Parser, error) {
var errs error
for _, parser := range parsers {
ret, next, err := parser(key, value)
if err == nil {
return ret, next, nil
}
errs = errors.Join(errs, err)
}
return nil, nil, fmt.Errorf("no parser succeeded: %w", errs)
}
}
func WithFallback(parser Parser, fallback FallbackParser) Parser {
if parser == nil {
return fallback.ToParser()
}
return func(key, value []byte) (SchemaEntry, Parser, error) {
entry, next, err := parser(key, value)
if err == nil {
return entry, WithFallback(next, fallback), nil
}
return fallback.ToParser()(key, value)
}
}
func (fp FallbackParser) ToParser() Parser {
return func(key, value []byte) (SchemaEntry, Parser, error) {
entry, next := fp(key, value)
return entry, next, nil
}
}
func (p Parser) ToFallbackParser() FallbackParser {
return func(key, value []byte) (SchemaEntry, Parser) {
entry, next, err := p(key, value)
if err != nil {
panic(fmt.Errorf(
"couldn't use that parser as a fallback parser, it returned an error: %w", err,
))
}
return entry, next
}
}

View file

@ -0,0 +1,29 @@
package buckets
import (
"github.com/davecgh/go-spew/spew"
)
func (b *PrefixBucket) DetailedString() string {
return spew.Sdump(*b)
}
func (b *PrefixContainerBucket) DetailedString() string {
return spew.Sdump(*b)
}
func (b *UserBucket) DetailedString() string {
return spew.Sdump(*b)
}
func (b *ContainerBucket) DetailedString() string {
return spew.Sdump(*b)
}
func (b *UserAttributeKeyBucket) DetailedString() string {
return spew.Sdump(*b)
}
func (b *UserAttributeValueBucket) DetailedString() string {
return spew.Sdump(*b)
}

View file

@ -0,0 +1,81 @@
package buckets
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
)
func (b *PrefixBucket) Filter(typ string, _ any) common.FilterResult {
switch typ {
case "cid":
return b.resolvers.cidResolver(false)
case "oid":
return b.resolvers.oidResolver(false)
default:
return common.No
}
}
func (b *PrefixContainerBucket) Filter(typ string, val any) common.FilterResult {
switch typ {
case "cid":
id := val.(cid.ID)
return b.resolvers.cidResolver(b.id.Equals(id))
case "oid":
return b.resolvers.oidResolver(false)
default:
return common.No
}
}
func (b *UserBucket) Filter(typ string, _ any) common.FilterResult {
switch typ {
case "cid":
return b.resolvers.cidResolver(false)
case "oid":
return b.resolvers.oidResolver(false)
default:
return common.No
}
}
func (b *ContainerBucket) Filter(typ string, val any) common.FilterResult {
switch typ {
case "cid":
id := val.(cid.ID)
return b.resolvers.cidResolver(b.id.Equals(id))
case "oid":
return b.resolvers.oidResolver(false)
default:
return common.No
}
}
func (b *UserAttributeKeyBucket) Filter(typ string, val any) common.FilterResult {
switch typ {
case "cid":
id := val.(cid.ID)
return common.IfThenElse(b.id.Equals(id), common.Yes, common.No)
case "oid":
return common.Maybe
case "key":
key := val.(string)
return common.IfThenElse(b.key == key, common.Yes, common.No)
case "value":
return common.Maybe
default:
return common.No
}
}
func (b *UserAttributeValueBucket) Filter(typ string, val any) common.FilterResult {
switch typ {
case "oid":
return common.Maybe
case "value":
value := val.(string)
return common.IfThenElse(b.value == value, common.Yes, common.No)
default:
return common.No
}
}

View file

@ -0,0 +1,111 @@
package buckets
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase/records"
)
var (
GraveyardParser = NewPrefixBucketParser(Graveyard, records.GraveyardRecordParser, Resolvers{
cidResolver: LenientResolver,
oidResolver: LenientResolver,
})
GarbageParser = NewPrefixBucketParser(Garbage, records.GarbageRecordParser, Resolvers{
cidResolver: LenientResolver,
oidResolver: LenientResolver,
})
ContainerVolumeParser = NewPrefixBucketParser(ContainerVolume, records.ContainerVolumeRecordParser, Resolvers{
cidResolver: LenientResolver,
oidResolver: StrictResolver,
})
LockedParser = NewPrefixBucketParser(
Locked,
NewContainerBucketParser(
records.LockedRecordParser,
Resolvers{
cidResolver: StrictResolver,
oidResolver: LenientResolver,
},
),
Resolvers{
cidResolver: LenientResolver,
oidResolver: LenientResolver,
},
)
ShardInfoParser = NewPrefixBucketParser(ShardInfo, records.ShardInfoRecordParser, Resolvers{
cidResolver: StrictResolver,
oidResolver: StrictResolver,
})
PrimaryParser = NewPrefixContainerBucketParser(Primary, records.ObjectRecordParser, Resolvers{
cidResolver: StrictResolver,
oidResolver: LenientResolver,
})
LockersParser = NewPrefixContainerBucketParser(Lockers, records.ObjectRecordParser, Resolvers{
cidResolver: StrictResolver,
oidResolver: LenientResolver,
})
TombstoneParser = NewPrefixContainerBucketParser(Tombstone, records.ObjectRecordParser, Resolvers{
cidResolver: StrictResolver,
oidResolver: LenientResolver,
})
SmallParser = NewPrefixContainerBucketParser(Small, records.SmallRecordParser, Resolvers{
cidResolver: StrictResolver,
oidResolver: LenientResolver,
})
RootParser = NewPrefixContainerBucketParser(Root, records.RootRecordParser, Resolvers{
cidResolver: StrictResolver,
oidResolver: LenientResolver,
})
OwnerParser = NewPrefixContainerBucketParser(
Owner,
NewUserBucketParser(
records.OwnerRecordParser,
Resolvers{
cidResolver: StrictResolver,
oidResolver: LenientResolver,
},
),
Resolvers{
cidResolver: StrictResolver,
oidResolver: LenientResolver,
},
)
UserAttributeParser = NewUserAttributeKeyBucketParser(
NewUserAttributeValueBucketParser(records.UserAttributeRecordParser),
)
PayloadHashParser = NewPrefixContainerBucketParser(PayloadHash, records.PayloadHashRecordParser, Resolvers{
cidResolver: StrictResolver,
oidResolver: StrictResolver,
})
ParentParser = NewPrefixContainerBucketParser(Parent, records.ParentRecordParser, Resolvers{
cidResolver: StrictResolver,
oidResolver: LenientResolver,
})
SplitParser = NewPrefixContainerBucketParser(Split, records.SplitRecordParser, Resolvers{
cidResolver: StrictResolver,
oidResolver: StrictResolver,
})
ContainerCountersParser = NewPrefixBucketParser(ContainerCounters, records.ContainerCountersRecordParser, Resolvers{
cidResolver: LenientResolver,
oidResolver: StrictResolver,
})
ECInfoParser = NewPrefixContainerBucketParser(ECInfo, records.ECInfoRecordParser, Resolvers{
cidResolver: StrictResolver,
oidResolver: LenientResolver,
})
)

View file

@ -0,0 +1,53 @@
package buckets
type Prefix byte
const (
Graveyard Prefix = iota
Garbage
ToMoveIt
ContainerVolume
Locked
ShardInfo
Primary
Lockers
_
Tombstone
Small
Root
Owner
UserAttribute
PayloadHash
Parent
Split
ContainerCounters
ECInfo
)
var x = map[Prefix]string{
Graveyard: "Graveyard",
Garbage: "Garbage",
ToMoveIt: "To Move It",
ContainerVolume: "Container Volume",
Locked: "Locked",
ShardInfo: "Shard Info",
Primary: "Primary",
Lockers: "Lockers",
Tombstone: "Tombstone",
Small: "Small",
Root: "Root",
Owner: "Owner",
UserAttribute: "User Attribute",
PayloadHash: "Payload Hash",
Parent: "Parent",
Split: "Split",
ContainerCounters: "Container Counters",
ECInfo: "EC Info",
}
func (p Prefix) String() string {
if s, ok := x[p]; ok {
return s
}
return "Unknown Prefix"
}

View file

@ -0,0 +1,48 @@
package buckets
import (
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
"github.com/gdamore/tcell/v2"
)
func (b *PrefixBucket) String() string {
return common.FormatSimple(
fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime,
)
}
func (b *PrefixContainerBucket) String() string {
return fmt.Sprintf(
"%s CID %s",
common.FormatSimple(
fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime,
),
common.FormatSimple(b.id.String(), tcell.ColorAqua),
)
}
func (b *UserBucket) String() string {
return "UID " + common.FormatSimple(b.id.String(), tcell.ColorAqua)
}
func (b *ContainerBucket) String() string {
return "CID " + common.FormatSimple(b.id.String(), tcell.ColorAqua)
}
func (b *UserAttributeKeyBucket) String() string {
return fmt.Sprintf("%s CID %s ATTR-KEY %s",
common.FormatSimple(
fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime,
),
common.FormatSimple(
fmt.Sprintf("%-44s", b.id), tcell.ColorAqua,
),
common.FormatSimple(b.key, tcell.ColorAqua),
)
}
func (b *UserAttributeValueBucket) String() string {
return "ATTR-VALUE " + common.FormatSimple(b.value, tcell.ColorAqua)
}

View file

@ -0,0 +1,166 @@
package buckets
import (
"errors"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/mr-tron/base58"
)
type (
PrefixBucket struct {
prefix Prefix
resolvers Resolvers
}
PrefixContainerBucket struct {
prefix Prefix
id cid.ID
resolvers Resolvers
}
ContainerBucket struct {
id cid.ID
resolvers Resolvers
}
UserBucket struct {
id user.ID
resolvers Resolvers
}
UserAttributeKeyBucket struct {
prefix Prefix
id cid.ID
key string
}
UserAttributeValueBucket struct {
value string
}
)
type (
FilterResolver = func(result bool) common.FilterResult
Resolvers struct {
cidResolver FilterResolver
oidResolver FilterResolver
}
)
var (
StrictResolver = func(x bool) common.FilterResult { return common.IfThenElse(x, common.Yes, common.No) }
LenientResolver = func(x bool) common.FilterResult { return common.IfThenElse(x, common.Yes, common.Maybe) }
)
var (
ErrNotBucket = errors.New("not a bucket")
ErrInvalidKeyLength = errors.New("invalid key length")
ErrInvalidValueLength = errors.New("invalid value length")
ErrInvalidPrefix = errors.New("invalid prefix")
)
func NewPrefixBucketParser(prefix Prefix, next common.Parser, resolvers Resolvers) common.Parser {
return func(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if value != nil {
return nil, nil, ErrNotBucket
}
if len(key) != 1 {
return nil, nil, ErrInvalidKeyLength
}
var b PrefixBucket
if b.prefix = Prefix(key[0]); b.prefix != prefix {
return nil, nil, ErrInvalidPrefix
}
b.resolvers = resolvers
return &b, next, nil
}
}
func NewPrefixContainerBucketParser(prefix Prefix, next common.Parser, resolvers Resolvers) common.Parser {
return func(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if value != nil {
return nil, nil, ErrNotBucket
}
if len(key) != 33 {
return nil, nil, ErrInvalidKeyLength
}
var b PrefixContainerBucket
if b.prefix = Prefix(key[0]); b.prefix != prefix {
return nil, nil, ErrInvalidPrefix
}
if err := b.id.Decode(key[1:]); err != nil {
return nil, nil, err
}
b.resolvers = resolvers
return &b, next, nil
}
}
func NewUserBucketParser(next common.Parser, resolvers Resolvers) common.Parser {
return func(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if value != nil {
return nil, nil, ErrNotBucket
}
var b UserBucket
if err := b.id.DecodeString(base58.Encode(key)); err != nil {
return nil, nil, err
}
b.resolvers = resolvers
return &b, next, nil
}
}
func NewContainerBucketParser(next common.Parser, resolvers Resolvers) common.Parser {
return func(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if value != nil {
return nil, nil, ErrNotBucket
}
if len(key) != 32 {
return nil, nil, ErrInvalidKeyLength
}
var b ContainerBucket
if err := b.id.Decode(key); err != nil {
return nil, nil, err
}
b.resolvers = resolvers
return &b, next, nil
}
}
func NewUserAttributeKeyBucketParser(next common.Parser) common.Parser {
return func(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if value != nil {
return nil, nil, ErrNotBucket
}
if len(key) < 34 {
return nil, nil, ErrInvalidKeyLength
}
var b UserAttributeKeyBucket
if b.prefix = Prefix(key[0]); b.prefix != UserAttribute {
return nil, nil, ErrInvalidPrefix
}
if err := b.id.Decode(key[1:33]); err != nil {
return nil, nil, err
}
b.key = string(key[33:])
return &b, next, nil
}
}
func NewUserAttributeValueBucketParser(next common.Parser) common.Parser {
return func(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if value != nil {
return nil, nil, ErrNotBucket
}
if len(key) == 0 {
return nil, nil, ErrInvalidKeyLength
}
var b UserAttributeValueBucket
b.value = string(key)
return &b, next, nil
}
}

View file

@ -0,0 +1,29 @@
package metabase
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase/buckets"
)
var MetabaseParser = common.WithFallback(
common.Any(
buckets.GraveyardParser,
buckets.GarbageParser,
buckets.ContainerVolumeParser,
buckets.LockedParser,
buckets.ShardInfoParser,
buckets.PrimaryParser,
buckets.LockersParser,
buckets.TombstoneParser,
buckets.SmallParser,
buckets.RootParser,
buckets.OwnerParser,
buckets.UserAttributeParser,
buckets.PayloadHashParser,
buckets.ParentParser,
buckets.SplitParser,
buckets.ContainerCountersParser,
buckets.ECInfoParser,
),
common.RawParser.ToFallbackParser(),
)

View file

@ -0,0 +1,65 @@
package records
import (
"github.com/davecgh/go-spew/spew"
)
func (r *GraveyardRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *GarbageRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *ContainerVolumeRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *LockedRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *ShardInfoRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *ObjectRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *SmallRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *RootRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *OwnerRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *UserAttributeRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *PayloadHashRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *ParentRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *SplitRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *ContainerCountersRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (r *ECInfoRecord) DetailedString() string {
return spew.Sdump(*r)
}

View file

@ -0,0 +1,145 @@
package records
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
func (r *GraveyardRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "cid":
id := val.(cid.ID)
return common.IfThenElse(r.object.Container().Equals(id), common.Yes, common.No)
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.object.Object().Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *GarbageRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "cid":
id := val.(cid.ID)
return common.IfThenElse(r.addr.Container().Equals(id), common.Yes, common.No)
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.addr.Object().Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *ContainerVolumeRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "cid":
id := val.(cid.ID)
return common.IfThenElse(r.id.Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *ShardInfoRecord) Filter(string, any) common.FilterResult {
return common.No
}
func (r *LockedRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.id.Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *ObjectRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.id.Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *SmallRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.id.Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *RootRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.id.Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *OwnerRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.id.Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *UserAttributeRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.id.Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *PayloadHashRecord) Filter(string, any) common.FilterResult {
return common.No
}
func (r *ParentRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.parent.Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *SplitRecord) Filter(string, any) common.FilterResult {
return common.No
}
func (r *ContainerCountersRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "cid":
id := val.(cid.ID)
return common.IfThenElse(r.id.Equals(id), common.Yes, common.No)
default:
return common.No
}
}
func (r *ECInfoRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.id.Equals(id), common.Yes, common.No)
default:
return common.No
}
}

View file

@ -0,0 +1,251 @@
package records
import (
"encoding/binary"
"errors"
"strconv"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
var (
ErrInvalidKeyLength = errors.New("invalid key length")
ErrInvalidValueLength = errors.New("invalid value length")
ErrInvalidPrefix = errors.New("invalid prefix")
)
func GraveyardRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if len(key) != 64 {
return nil, nil, ErrInvalidKeyLength
}
if len(value) != 64 {
return nil, nil, ErrInvalidValueLength
}
var (
cnr cid.ID
obj oid.ID
r GraveyardRecord
)
_ = cnr.Decode(key[:32])
_ = obj.Decode(key[32:])
r.object.SetContainer(cnr)
r.object.SetObject(obj)
_ = cnr.Decode(value[:32])
_ = obj.Decode(value[32:])
r.tombstone.SetContainer(cnr)
r.tombstone.SetObject(obj)
return &r, nil, nil
}
func GarbageRecordParser(key, _ []byte) (common.SchemaEntry, common.Parser, error) {
if len(key) != 64 {
return nil, nil, ErrInvalidKeyLength
}
var (
cnr cid.ID
obj oid.ID
r GarbageRecord
)
_ = cnr.Decode(key[:32])
_ = obj.Decode(key[32:])
r.addr.SetContainer(cnr)
r.addr.SetObject(obj)
return &r, nil, nil
}
func ContainerVolumeRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if len(key) != 32 {
return nil, nil, ErrInvalidKeyLength
}
if len(value) != 8 {
return nil, nil, ErrInvalidValueLength
}
var r ContainerVolumeRecord
_ = r.id.Decode(key)
r.volume = binary.LittleEndian.Uint64(value)
return &r, nil, nil
}
func LockedRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
var (
r LockedRecord
err error
)
if err := r.id.Decode(key); err != nil {
return nil, nil, err
}
if r.ids, err = DecodeOIDs(value); err != nil {
return nil, nil, err
}
return &r, nil, nil
}
func ShardInfoRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if len(key) == 0 {
return nil, nil, ErrInvalidKeyLength
}
var r ShardInfoRecord
if string(key) == "id" {
r.label = string(key)
r.value = shard.ID(value).String()
return &r, nil, nil
}
if len(value) != 8 {
return nil, nil, ErrInvalidValueLength
}
r.label = string(key)
r.value = strconv.FormatUint(binary.LittleEndian.Uint64(value), 10)
return &r, nil, nil
}
func ObjectRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if len(key) != 32 {
return nil, nil, ErrInvalidKeyLength
}
var r ObjectRecord
_ = r.id.Decode(key)
if err := r.object.Unmarshal(value); err != nil {
return nil, nil, err
}
return &r, nil, nil
}
func SmallRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
var r SmallRecord
if err := r.id.Decode(key); err != nil {
return nil, nil, err
}
if len(value) != 0 {
x := string(value)
r.storageID = &x
}
return &r, nil, nil
}
func RootRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
var r RootRecord
if err := r.id.Decode(key); err != nil {
return nil, nil, err
}
if len(value) == 0 {
return &r, nil, nil
}
r.info = &objectSDK.SplitInfo{}
if err := r.info.Unmarshal(value); err != nil {
return nil, nil, err
}
return &r, nil, nil
}
func OwnerRecordParser(key, _ []byte) (common.SchemaEntry, common.Parser, error) {
var r OwnerRecord
if err := r.id.Decode(key); err != nil {
return nil, nil, err
}
return &r, nil, nil
}
func UserAttributeRecordParser(key, _ []byte) (common.SchemaEntry, common.Parser, error) {
var r UserAttributeRecord
if err := r.id.Decode(key); err != nil {
return nil, nil, err
}
return &r, nil, nil
}
func PayloadHashRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if len(key) != 32 {
return nil, nil, ErrInvalidKeyLength
}
var (
err error
r PayloadHashRecord
)
r.checksum.SetSHA256([32]byte(key))
if r.ids, err = DecodeOIDs(value); err != nil {
return nil, nil, err
}
return &r, nil, nil
}
func ParentRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
var (
r ParentRecord
err error
)
if err = r.parent.Decode(key); err != nil {
return nil, nil, err
}
if r.ids, err = DecodeOIDs(value); err != nil {
return nil, nil, err
}
return &r, nil, nil
}
func SplitRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
var (
err error
r SplitRecord
)
if err = r.id.UnmarshalBinary(key); err != nil {
return nil, nil, err
}
if r.ids, err = DecodeOIDs(value); err != nil {
return nil, nil, err
}
return &r, nil, nil
}
func ContainerCountersRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if len(value) != 24 {
return nil, nil, ErrInvalidValueLength
}
var r ContainerCountersRecord
if err := r.id.Decode(key); err != nil {
return nil, nil, err
}
r.logical = binary.LittleEndian.Uint64(value[:8])
r.physical = binary.LittleEndian.Uint64(value[8:16])
r.user = binary.LittleEndian.Uint64(value[16:24])
return &r, nil, nil
}
func ECInfoRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
var (
r ECInfoRecord
err error
)
if err := r.id.Decode(key); err != nil {
return nil, nil, err
}
if r.ids, err = DecodeOIDs(value); err != nil {
return nil, nil, err
}
return &r, nil, nil
}

View file

@ -0,0 +1,135 @@
package records
import (
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
)
func (r *GraveyardRecord) String() string {
return fmt.Sprintf(
"Object CID %s OID %s %c Tombstone CID %s OID %s",
common.FormatSimple(fmt.Sprintf("%-44s", r.object.Container()), tcell.ColorAqua),
common.FormatSimple(fmt.Sprintf("%-44s", r.object.Object()), tcell.ColorAqua),
tview.Borders.Vertical,
common.FormatSimple(fmt.Sprintf("%-44s", r.tombstone.Container()), tcell.ColorAqua),
common.FormatSimple(fmt.Sprintf("%-44s", r.tombstone.Object()), tcell.ColorAqua),
)
}
func (r *GarbageRecord) String() string {
return fmt.Sprintf(
"CID %-44s OID %-44s",
common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Container()), tcell.ColorAqua),
common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Object()), tcell.ColorAqua),
)
}
func (r *ContainerVolumeRecord) String() string {
return fmt.Sprintf(
"CID %-44s %c %d",
common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua),
tview.Borders.Vertical,
r.volume,
)
}
func (r *LockedRecord) String() string {
return fmt.Sprintf(
"Locker OID %s %c Locked [%d]OID {...}",
common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua),
tview.Borders.Vertical,
len(r.ids),
)
}
func (r *ShardInfoRecord) String() string {
return fmt.Sprintf("%-13s %c %s", r.label, tview.Borders.Vertical, r.value)
}
func (r *ObjectRecord) String() string {
return fmt.Sprintf(
"OID %s %c Object {...}",
common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua),
tview.Borders.Vertical,
)
}
func (r *SmallRecord) String() string {
s := fmt.Sprintf(
"OID %s %c",
common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua),
tview.Borders.Vertical,
)
if r.storageID != nil {
s = fmt.Sprintf("%s %s", s, *r.storageID)
}
return s
}
func (r *RootRecord) String() string {
s := fmt.Sprintf(
"Root OID %s %c",
common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua),
tview.Borders.Vertical,
)
if r.info != nil {
s += " Split info {...}"
}
return s
}
func (r *OwnerRecord) String() string {
return "OID " + common.FormatSimple(r.id.String(), tcell.ColorAqua)
}
func (r *UserAttributeRecord) String() string {
return "OID " + common.FormatSimple(r.id.String(), tcell.ColorAqua)
}
func (r *PayloadHashRecord) String() string {
return fmt.Sprintf(
"Checksum %s %c [%d]OID {...}",
common.FormatSimple(r.checksum.String(), tcell.ColorAqua),
tview.Borders.Vertical,
len(r.ids),
)
}
func (r *ParentRecord) String() string {
return fmt.Sprintf(
"Parent OID %s %c [%d]OID {...}",
common.FormatSimple(fmt.Sprintf("%-44s", r.parent), tcell.ColorAqua),
tview.Borders.Vertical,
len(r.ids),
)
}
func (r *SplitRecord) String() string {
return fmt.Sprintf(
"Split ID %s %c [%d]OID {...}",
common.FormatSimple(r.id.String(), tcell.ColorAqua),
tview.Borders.Vertical,
len(r.ids),
)
}
func (r *ContainerCountersRecord) String() string {
return fmt.Sprintf(
"CID %s %c logical %d, physical %d, user %d",
common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua),
tview.Borders.Vertical,
r.logical, r.physical, r.user,
)
}
func (r *ECInfoRecord) String() string {
return fmt.Sprintf(
"OID %s %c [%d]OID {...}",
common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua),
tview.Borders.Vertical,
len(r.ids),
)
}

View file

@ -0,0 +1,82 @@
package records
import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/google/uuid"
)
type (
GraveyardRecord struct {
object, tombstone oid.Address
}
GarbageRecord struct {
addr oid.Address
}
ContainerVolumeRecord struct {
id cid.ID
volume uint64
}
LockedRecord struct {
id oid.ID
ids []oid.ID
}
ShardInfoRecord struct {
label string
value string
}
ObjectRecord struct {
id oid.ID
object objectSDK.Object
}
SmallRecord struct {
id oid.ID
storageID *string // optional
}
RootRecord struct {
id oid.ID
info *objectSDK.SplitInfo // optional
}
OwnerRecord struct {
id oid.ID
}
UserAttributeRecord struct {
id oid.ID
}
PayloadHashRecord struct {
checksum checksum.Checksum
ids []oid.ID
}
ParentRecord struct {
parent oid.ID
ids []oid.ID
}
SplitRecord struct {
id uuid.UUID
ids []oid.ID
}
ContainerCountersRecord struct {
id cid.ID
logical, physical, user uint64
}
ECInfoRecord struct {
id oid.ID
ids []oid.ID
}
)

View file

@ -0,0 +1,20 @@
package records
import (
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/nspcc-dev/neo-go/pkg/io"
)
func DecodeOIDs(data []byte) ([]oid.ID, error) {
r := io.NewBinReaderFromBuf(data)
size := r.ReadVarUint()
oids := make([]oid.ID, size)
for i := uint64(0); i < size; i++ {
if err := oids[i].Decode(r.ReadVarBytes()); err != nil {
return nil, err
}
}
return oids, nil
}

View file

@ -0,0 +1,63 @@
package writecache
import (
"bytes"
"errors"
"fmt"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/mr-tron/base58"
)
var WritecacheParser = common.WithFallback(
DefaultBucketParser,
common.RawParser.ToFallbackParser(),
)
func DefaultBucketParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
if value != nil {
return nil, nil, errors.New("not a bucket")
}
if !bytes.Equal(key, []byte{0}) {
return nil, nil, errors.New("invalid key")
}
return &DefaultBucket{}, DefaultRecordParser, nil
}
func DefaultRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 2 {
return nil, nil, errors.New("invalid key, expected address string <CID>/<OID>")
}
cnrRaw, err := base58.Decode(parts[0])
if err != nil {
return nil, nil, errors.New("can't decode CID string")
}
objRaw, err := base58.Decode(parts[1])
if err != nil {
return nil, nil, errors.New("can't decode OID string")
}
cnr := cid.ID{}
if err := cnr.Decode(cnrRaw); err != nil {
return nil, nil, fmt.Errorf("can't decode CID: %w", err)
}
obj := oid.ID{}
if err := obj.Decode(objRaw); err != nil {
return nil, nil, fmt.Errorf("can't decode OID: %w", err)
}
var r DefaultRecord
r.addr.SetContainer(cnr)
r.addr.SetObject(obj)
r.data = value[:]
return &r, nil, nil
}

View file

@ -0,0 +1,66 @@
package writecache
import (
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/davecgh/go-spew/spew"
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
)
type (
DefaultBucket struct{}
DefaultRecord struct {
addr oid.Address
data []byte
}
)
func (b *DefaultBucket) String() string {
return common.FormatSimple("0 Default", tcell.ColorLime)
}
func (r *DefaultRecord) String() string {
return fmt.Sprintf(
"CID %s OID %s %c Data {...}",
common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Container()), tcell.ColorAqua),
common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Object()), tcell.ColorAqua),
tview.Borders.Vertical,
)
}
func (b *DefaultBucket) DetailedString() string {
return spew.Sdump(*b)
}
func (r *DefaultRecord) DetailedString() string {
return spew.Sdump(*r)
}
func (b *DefaultBucket) Filter(typ string, _ any) common.FilterResult {
switch typ {
case "cid":
return common.Maybe
case "oid":
return common.Maybe
default:
return common.No
}
}
func (r *DefaultRecord) Filter(typ string, val any) common.FilterResult {
switch typ {
case "cid":
id := val.(cid.ID)
return common.IfThenElse(r.addr.Container().Equals(id), common.Yes, common.No)
case "oid":
id := val.(oid.ID)
return common.IfThenElse(r.addr.Object().Equals(id), common.Yes, common.No)
default:
return common.No
}
}

View file

@ -0,0 +1,257 @@
package tui
import (
"context"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
)
type BucketsView struct {
*tview.Box
mu sync.Mutex
view *tview.TreeView
nodeToUpdate *tview.TreeNode
ui *UI
filter *Filter
}
type bucketNode struct {
bucket *Bucket
filter *Filter
}
func NewBucketsView(ui *UI, filter *Filter) *BucketsView {
return &BucketsView{
Box: tview.NewBox(),
view: tview.NewTreeView(),
ui: ui,
filter: filter,
}
}
func (v *BucketsView) Mount(_ context.Context) error {
root := tview.NewTreeNode(".")
root.SetExpanded(false)
root.SetSelectable(false)
root.SetReference(&bucketNode{
bucket: &Bucket{NextParser: v.ui.rootParser},
filter: v.filter,
})
v.nodeToUpdate = root
v.view.SetRoot(root)
v.view.SetCurrentNode(root)
return nil
}
func (v *BucketsView) Update(ctx context.Context) error {
if v.nodeToUpdate == nil {
return nil
}
defer func() { v.nodeToUpdate = nil }()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
ready := make(chan struct{})
errCh := make(chan error)
tmp := tview.NewTreeNode(v.nodeToUpdate.GetText())
tmp.SetReference(v.nodeToUpdate.GetReference())
node := v.nodeToUpdate.GetReference().(*bucketNode)
go func() {
defer close(ready)
hasBuckets, err := HasBuckets(ctx, v.ui.db, node.bucket.Path)
if err != nil {
errCh <- err
}
// Show the selected bucket's records instead.
if !hasBuckets && node.bucket.NextParser != nil {
v.ui.moveNextPage(NewRecordsView(v.ui, node.bucket, node.filter))
}
if v.nodeToUpdate.IsExpanded() {
return
}
err = v.loadNodeChildren(ctx, tmp, node.filter)
if err != nil {
errCh <- err
}
}()
select {
case <-ctx.Done():
case <-ready:
v.mu.Lock()
v.nodeToUpdate.SetChildren(tmp.GetChildren())
v.nodeToUpdate.SetExpanded(!v.nodeToUpdate.IsExpanded())
v.mu.Unlock()
case err := <-errCh:
return err
}
return nil
}
func (v *BucketsView) Unmount() {
}
func (v *BucketsView) Draw(screen tcell.Screen) {
x, y, width, height := v.GetInnerRect()
v.view.SetRect(x, y, width, height)
v.view.Draw(screen)
}
func (v *BucketsView) loadNodeChildren(
ctx context.Context, node *tview.TreeNode, filter *Filter,
) error {
parentBucket := node.GetReference().(*bucketNode).bucket
path := parentBucket.Path
parser := parentBucket.NextParser
buffer, err := LoadBuckets(ctx, v.ui.db, path, v.ui.loadBufferSize)
if err != nil {
return err
}
for item := range buffer {
if item.err != nil {
return item.err
}
bucket := item.val
bucket.Entry, bucket.NextParser, err = parser(bucket.Name, nil)
if err != nil {
return err
}
satisfies, err := v.bucketSatisfiesFilter(ctx, bucket, filter)
if err != nil {
return err
}
if !satisfies {
continue
}
child := tview.NewTreeNode(bucket.Entry.String()).
SetSelectable(true).
SetExpanded(false).
SetReference(&bucketNode{
bucket: bucket,
filter: filter.Apply(bucket.Entry),
})
node.AddChild(child)
}
return nil
}
func (v *BucketsView) bucketSatisfiesFilter(
ctx context.Context, bucket *Bucket, filter *Filter,
) (bool, error) {
// Does the current bucket satisfies the filter?
filter = filter.Apply(bucket.Entry)
if filter.Result() == common.Yes {
return true, nil
}
if filter.Result() == common.No {
return false, nil
}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// Check the current bucket's nested buckets if exist
bucketsBuffer, err := LoadBuckets(ctx, v.ui.db, bucket.Path, v.ui.loadBufferSize)
if err != nil {
return false, err
}
for item := range bucketsBuffer {
if item.err != nil {
return false, item.err
}
b := item.val
b.Entry, b.NextParser, err = bucket.NextParser(b.Name, nil)
if err != nil {
return false, err
}
satisfies, err := v.bucketSatisfiesFilter(ctx, b, filter)
if err != nil {
return false, err
}
if satisfies {
return true, nil
}
}
// Check the current bucket's nested records if exist
recordsBuffer, err := LoadRecords(ctx, v.ui.db, bucket.Path, v.ui.loadBufferSize)
if err != nil {
return false, err
}
for item := range recordsBuffer {
if item.err != nil {
return false, item.err
}
r := item.val
r.Entry, _, err = bucket.NextParser(r.Key, r.Value)
if err != nil {
return false, err
}
if filter.Apply(r.Entry).Result() == common.Yes {
return true, nil
}
}
return false, nil
}
func (v *BucketsView) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) {
return v.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) {
currentNode := v.view.GetCurrentNode()
if currentNode == nil {
return
}
switch event.Key() {
case tcell.KeyEnter:
// Expand or collapse the selected bucket's nested buckets,
// otherwise, navigate to that bucket's records.
v.nodeToUpdate = currentNode
case tcell.KeyCtrlR:
// Navigate to the selected bucket's records.
bucketNode := currentNode.GetReference().(*bucketNode)
v.ui.moveNextPage(NewRecordsView(v.ui, bucketNode.bucket, bucketNode.filter))
case tcell.KeyCtrlD:
// Navigate to the selected bucket's detailed view.
bucketNode := currentNode.GetReference().(*bucketNode)
v.ui.moveNextPage(NewDetailedView(bucketNode.bucket.Entry.DetailedString()))
default:
v.view.InputHandler()(event, func(tview.Primitive) {})
}
})
}

View file

@ -0,0 +1,160 @@
package tui
import (
"context"
"errors"
"fmt"
"go.etcd.io/bbolt"
)
type Item[T any] struct {
val T
err error
}
func resolvePath(tx *bbolt.Tx, path [][]byte) (*bbolt.Bucket, error) {
if len(path) == 0 {
return nil, errors.New("can't find bucket without path")
}
name := path[0]
bucket := tx.Bucket(name)
if bucket == nil {
return nil, fmt.Errorf("no bucket with name %s", name)
}
for _, name := range path[1:] {
bucket = bucket.Bucket(name)
if bucket == nil {
return nil, fmt.Errorf("no bucket with name %s", name)
}
}
return bucket, nil
}
func load[T any](
ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int,
filter func(key, value []byte) bool, transform func(key, value []byte) T,
) (<-chan Item[T], error) {
buffer := make(chan Item[T], bufferSize)
go func() {
defer close(buffer)
err := db.View(func(tx *bbolt.Tx) error {
var cursor *bbolt.Cursor
if len(path) == 0 {
cursor = tx.Cursor()
} else {
bucket, err := resolvePath(tx, path)
if err != nil {
buffer <- Item[T]{err: fmt.Errorf("can't find bucket: %w", err)}
return nil
}
cursor = bucket.Cursor()
}
key, value := cursor.First()
for {
if key == nil {
return nil
}
if filter != nil && !filter(key, value) {
key, value = cursor.Next()
continue
}
select {
case <-ctx.Done():
return nil
case buffer <- Item[T]{val: transform(key, value)}:
key, value = cursor.Next()
}
}
})
if err != nil {
buffer <- Item[T]{err: err}
}
}()
return buffer, nil
}
func LoadBuckets(
ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int,
) (<-chan Item[*Bucket], error) {
buffer, err := load(
ctx, db, path, bufferSize,
func(_, value []byte) bool {
return value == nil
},
func(key, _ []byte) *Bucket {
base := make([][]byte, 0, len(path))
base = append(base, path...)
return &Bucket{
Name: key,
Path: append(base, key),
}
},
)
if err != nil {
return nil, fmt.Errorf("can't start iterating bucket: %w", err)
}
return buffer, nil
}
func LoadRecords(
ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int,
) (<-chan Item[*Record], error) {
buffer, err := load(
ctx, db, path, bufferSize,
func(_, value []byte) bool {
return value != nil
},
func(key, value []byte) *Record {
base := make([][]byte, 0, len(path))
base = append(base, path...)
return &Record{
Key: key,
Value: value,
Path: append(base, key),
}
},
)
if err != nil {
return nil, fmt.Errorf("can't start iterating bucket: %w", err)
}
return buffer, nil
}
// HasBuckets checks if a bucket has nested buckets. It relies on assumption
// that a bucket can have either nested buckets or records but not both.
func HasBuckets(ctx context.Context, db *bbolt.DB, path [][]byte) (bool, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
buffer, err := load(
ctx, db, path, 1,
nil,
func(_, value []byte) []byte { return value },
)
if err != nil {
return false, err
}
x, ok := <-buffer
if !ok {
return false, nil
}
if x.err != nil {
return false, err
}
if x.val != nil {
return false, err
}
return true, nil
}

View file

@ -0,0 +1,24 @@
package tui
import (
"context"
"github.com/rivo/tview"
)
type DetailedView struct {
*tview.TextView
}
func NewDetailedView(detailed string) *DetailedView {
v := &DetailedView{
TextView: tview.NewTextView(),
}
v.SetDynamicColors(true)
v.SetText(detailed)
return v
}
func (v *DetailedView) Mount(_ context.Context) error { return nil }
func (v *DetailedView) Update(_ context.Context) error { return nil }
func (v *DetailedView) Unmount() {}

View file

@ -0,0 +1,44 @@
package tui
import (
"maps"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
)
type Filter struct {
values map[string]any
results map[string]common.FilterResult
}
func NewFilter(values map[string]any) *Filter {
f := &Filter{
values: maps.Clone(values),
results: make(map[string]common.FilterResult),
}
for tag := range values {
f.results[tag] = common.No
}
return f
}
func (f *Filter) Apply(e common.SchemaEntry) *Filter {
filter := &Filter{
values: f.values,
results: maps.Clone(f.results),
}
for tag, value := range filter.values {
filter.results[tag] = max(filter.results[tag], e.Filter(tag, value))
}
return filter
}
func (f *Filter) Result() common.FilterResult {
current := common.Yes
for _, r := range f.results {
current = min(r, current)
}
return current
}

View file

@ -0,0 +1,38 @@
[green::b]HOTKEYS[-::-]
[green::b]Navigation[-::-]
[yellow::b]Down Arrow[-::-] / [yellow::b]j[-::-]
Scroll down.
[yellow::b]Up Arrow[-::-] / [yellow::b]k[-::-]
Scroll up.
[yellow::b]Page Down[-::-] / [yellow::b]Ctrl-f[-::-]
Scroll down by a full page.
[yellow::b]Page Up[-::-] / [yellow::b]Ctrl-b[-::-]
Scroll up by a full page.
[green::b]Actions[-::-]
[yellow::b]Enter[-::-]
Perform actions based on the current context:
- In Buckets View:
- Expand/collapse the selected bucket to show/hide its nested buckets.
- If no nested buckets exist, navigate to the selected bucket's records.
- In Records View: Open the detailed view of the selected record.
[yellow::b]Escape[-::-]
Return to the previous page, opposite of [yellow::b]Enter[-::-].
Refer to the [green::b]SEARCHING[-::-] section for more specific actions.
[green::b]Alternative Action Hotkeys[-::-]
[yellow::b]Ctrl-r[-::-]
Directly navigate to the selected bucket's records.
[yellow::b]Ctrl-d[-::-]
Access the detailed view of the selected bucket.

View file

@ -0,0 +1,26 @@
[green::b]SEARCHING[-::-]
[green::b]Hotkeys[-::-]
[yellow::b]/[-::-]
Initiate the search prompt.
- The prompt follows this syntax: [yellow::b]tag:value [+ tag:value]...[-::-]
- Multiple filter can be combined with [yellow::b]+[-::-], the result is an intersection of those filters' result sets.
- Any leading and trailing whitespace will be ignored.
- An empty prompt will return all results with no filters applied.
- Refer to the [green::b]Available Search Filters[-::-] section below for a list of valid filter tags.
[yellow::b]Enter[-::-]
Execute the search based on the entered prompt.
- If the prompt is invalid, an error message will be displayed.
[yellow::b]Escape[-::-]
Exit the search prompt without performing a search.
[yellow::b]Down Arrow[-::-], [yellow::b]Up Arrow[-::-]
Scroll through the search history.
[green::b]Available Search Filters[-::-]
%s

View file

@ -0,0 +1,101 @@
package tui
import (
_ "embed"
"fmt"
"strings"
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
)
var (
//go:embed help-pages/hotkeys.txt
hotkeysHelpText string
//go:embed help-pages/searching.txt
searchingHelpText string
)
type HelpPage struct {
*tview.Box
pages []*tview.TextView
currentPage int
filters []string
filterHints map[string]string
}
func NewHelpPage(filters []string, hints map[string]string) *HelpPage {
hp := &HelpPage{
Box: tview.NewBox(),
filters: filters,
filterHints: hints,
}
page := tview.NewTextView().
SetDynamicColors(true).
SetText(hotkeysHelpText)
hp.addPage(page)
page = tview.NewTextView().
SetDynamicColors(true).
SetText(fmt.Sprintf(searchingHelpText, hp.getFiltersText()))
hp.addPage(page)
return hp
}
func (hp *HelpPage) addPage(page *tview.TextView) {
hp.pages = append(hp.pages, page)
}
func (hp *HelpPage) getFiltersText() string {
if len(hp.filters) == 0 {
return "\t\tNo filters defined.\n"
}
filtersText := strings.Builder{}
gapSize := 4
tagMaxWidth := 3
for _, filter := range hp.filters {
tagMaxWidth = max(tagMaxWidth, len(filter))
}
filtersText.WriteString("\t\t[yellow::b]Tag")
filtersText.WriteString(strings.Repeat(" ", gapSize))
filtersText.WriteString("\tValue[-::-]\n\n")
for _, filter := range hp.filters {
filtersText.WriteString("\t\t")
filtersText.WriteString(filter)
filtersText.WriteString(strings.Repeat(" ", tagMaxWidth-len(filter)+gapSize))
filtersText.WriteString(hp.filterHints[filter])
filtersText.WriteRune('\n')
}
return filtersText.String()
}
func (hp *HelpPage) Draw(screen tcell.Screen) {
x, y, width, height := hp.GetInnerRect()
hp.pages[hp.currentPage].SetRect(x+1, y+1, width-2, height-2)
hp.pages[hp.currentPage].Draw(screen)
}
func (hp *HelpPage) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) {
return hp.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) {
if event.Key() == tcell.KeyEnter {
hp.currentPage++
hp.currentPage %= len(hp.pages)
return
}
hp.pages[hp.currentPage].InputHandler()(event, func(tview.Primitive) {})
})
}
func (hp *HelpPage) MouseHandler() func(action tview.MouseAction, event *tcell.EventMouse, setFocus func(p tview.Primitive)) (consumed bool, capture tview.Primitive) {
return hp.WrapMouseHandler(func(action tview.MouseAction, event *tcell.EventMouse, _ func(tview.Primitive)) (consumed bool, capture tview.Primitive) {
return hp.pages[hp.currentPage].MouseHandler()(action, event, func(tview.Primitive) {})
})
}

View file

@ -0,0 +1,77 @@
package tui
import (
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
)
type InputFieldWithHistory struct {
*tview.InputField
history []string
historyLimit int
historyPointer int
currentContent string
}
func NewInputFieldWithHistory(historyLimit int) *InputFieldWithHistory {
return &InputFieldWithHistory{
InputField: tview.NewInputField(),
historyLimit: historyLimit,
}
}
func (f *InputFieldWithHistory) AddToHistory(s string) {
// Stop scrolling history on history change, need to start scrolling again.
defer func() { f.historyPointer = len(f.history) }()
// Used history data for search prompt, so just make that data recent.
if f.historyPointer != len(f.history) && s == f.history[f.historyPointer] {
f.history = append(f.history[:f.historyPointer], f.history[f.historyPointer+1:]...)
f.history = append(f.history, s)
}
if len(f.history) == f.historyLimit {
f.history = f.history[1:]
}
f.history = append(f.history, s)
}
func (f *InputFieldWithHistory) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) {
return f.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) {
switch event.Key() {
case tcell.KeyDown:
if len(f.history) == 0 {
return
}
// Need to start iterating before.
if f.historyPointer == len(f.history) {
return
}
// Iterate to most recent prompts.
f.historyPointer++
// Stop iterating over history.
if f.historyPointer == len(f.history) {
f.InputField.SetText(f.currentContent)
return
}
f.InputField.SetText(f.history[f.historyPointer])
case tcell.KeyUp:
if len(f.history) == 0 {
return
}
// Start iterating over history.
if f.historyPointer == len(f.history) {
f.currentContent = f.InputField.GetText()
}
// End of history.
if f.historyPointer == 0 {
return
}
// Iterate to least recent prompts.
f.historyPointer--
f.InputField.SetText(f.history[f.historyPointer])
default:
f.InputField.InputHandler()(event, func(tview.Primitive) {})
}
})
}

View file

@ -0,0 +1,72 @@
package tui
import (
"context"
"fmt"
"sync/atomic"
"time"
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
)
type LoadingBar struct {
*tview.Box
view *tview.TextView
secondsElapsed atomic.Int64
needDrawFunc func()
reset func()
}
func NewLoadingBar(needDrawFunc func()) *LoadingBar {
b := &LoadingBar{
Box: tview.NewBox(),
view: tview.NewTextView(),
needDrawFunc: needDrawFunc,
}
b.view.SetBackgroundColor(tview.Styles.PrimaryTextColor)
b.view.SetTextColor(b.GetBackgroundColor())
return b
}
func (b *LoadingBar) Start(ctx context.Context) {
ctx, b.reset = context.WithCancel(ctx)
go func() {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
b.secondsElapsed.Store(0)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
b.secondsElapsed.Add(1)
b.needDrawFunc()
}
}
}()
}
func (b *LoadingBar) Stop() {
b.reset()
}
func (b *LoadingBar) Draw(screen tcell.Screen) {
seconds := b.secondsElapsed.Load()
var time string
switch {
case seconds < 60:
time = fmt.Sprintf("%ds", seconds)
default:
time = fmt.Sprintf("%dm%ds", seconds/60, seconds%60)
}
b.view.SetText(fmt.Sprintf(" Loading... %s (press Escape to cancel) ", time))
x, y, width, _ := b.GetInnerRect()
b.view.SetRect(x, y, width, 1)
b.view.Draw(screen)
}

View file

@ -0,0 +1,271 @@
package tui
import (
"context"
"errors"
"fmt"
"math"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
)
type updateType int
const (
other updateType = iota
moveToPrevPage
moveToNextPage
moveUp
moveDown
moveHome
moveEnd
)
type RecordsView struct {
*tview.Box
mu sync.RWMutex
onUnmount func()
bucket *Bucket
records []*Record
buffer chan *Record
firstRecordIndex int
lastRecordIndex int
selectedRecordIndex int
updateType updateType
ui *UI
filter *Filter
}
func NewRecordsView(ui *UI, bucket *Bucket, filter *Filter) *RecordsView {
return &RecordsView{
Box: tview.NewBox(),
bucket: bucket,
ui: ui,
filter: filter,
}
}
func (v *RecordsView) Mount(ctx context.Context) error {
if v.onUnmount != nil {
return errors.New("try to mount already mounted component")
}
ctx, v.onUnmount = context.WithCancel(ctx)
tempBuffer, err := LoadRecords(ctx, v.ui.db, v.bucket.Path, v.ui.loadBufferSize)
if err != nil {
return err
}
v.buffer = make(chan *Record, v.ui.loadBufferSize)
go func() {
defer close(v.buffer)
for item := range tempBuffer {
if item.err != nil {
v.ui.stopOnError(err)
break
}
record := item.val
record.Entry, _, err = v.bucket.NextParser(record.Key, record.Value)
if err != nil {
v.ui.stopOnError(err)
break
}
if v.filter.Apply(record.Entry).Result() != common.Yes {
continue
}
v.buffer <- record
}
}()
return nil
}
func (v *RecordsView) Unmount() {
if v.onUnmount == nil {
panic("try to unmount not mounted component")
}
v.onUnmount()
v.onUnmount = nil
}
func (v *RecordsView) Update(ctx context.Context) error {
_, _, _, recordsPerPage := v.GetInnerRect()
firstRecordIndex, lastRecordIndex, selectedRecordIndex := v.getNewIndexes()
loop:
for len(v.records) < lastRecordIndex {
select {
case <-ctx.Done():
return nil
case record, ok := <-v.buffer:
if !ok {
break loop
}
v.records = append(v.records, record)
}
}
// Set the update type to its default value after some specific key event
// has been handled.
v.updateType = other
firstRecordIndex = max(0, min(firstRecordIndex, len(v.records)-recordsPerPage))
lastRecordIndex = min(firstRecordIndex+recordsPerPage, len(v.records))
selectedRecordIndex = min(selectedRecordIndex, lastRecordIndex-1)
v.mu.Lock()
v.firstRecordIndex = firstRecordIndex
v.lastRecordIndex = lastRecordIndex
v.selectedRecordIndex = selectedRecordIndex
v.mu.Unlock()
return nil
}
func (v *RecordsView) getNewIndexes() (int, int, int) {
v.mu.RLock()
firstRecordIndex := v.firstRecordIndex
lastRecordIndex := v.lastRecordIndex
selectedRecordIndex := v.selectedRecordIndex
v.mu.RUnlock()
_, _, _, recordsPerPage := v.GetInnerRect()
switch v.updateType {
case moveUp:
if selectedRecordIndex != firstRecordIndex {
selectedRecordIndex--
break
}
firstRecordIndex = max(0, firstRecordIndex-1)
lastRecordIndex = min(firstRecordIndex+recordsPerPage, len(v.records))
selectedRecordIndex = firstRecordIndex
case moveToPrevPage:
if selectedRecordIndex != firstRecordIndex {
selectedRecordIndex = firstRecordIndex
break
}
firstRecordIndex = max(0, firstRecordIndex-recordsPerPage)
lastRecordIndex = firstRecordIndex + recordsPerPage
selectedRecordIndex = firstRecordIndex
case moveDown:
if selectedRecordIndex != lastRecordIndex-1 {
selectedRecordIndex++
break
}
firstRecordIndex++
lastRecordIndex++
selectedRecordIndex++
case moveToNextPage:
if selectedRecordIndex != lastRecordIndex-1 {
selectedRecordIndex = lastRecordIndex - 1
break
}
firstRecordIndex += recordsPerPage
lastRecordIndex = firstRecordIndex + recordsPerPage
selectedRecordIndex = lastRecordIndex - 1
case moveHome:
firstRecordIndex = 0
lastRecordIndex = firstRecordIndex + recordsPerPage
selectedRecordIndex = 0
case moveEnd:
lastRecordIndex = math.MaxInt32
firstRecordIndex = lastRecordIndex - recordsPerPage
selectedRecordIndex = lastRecordIndex - 1
default:
lastRecordIndex = firstRecordIndex + recordsPerPage
}
return firstRecordIndex, lastRecordIndex, selectedRecordIndex
}
func (v *RecordsView) GetInnerRect() (int, int, int, int) {
x, y, width, height := v.Box.GetInnerRect()
// Left padding.
x = min(x+3, x+width-1)
width = max(width-3, 0)
return x, y, width, height
}
func (v *RecordsView) Draw(screen tcell.Screen) {
v.mu.RLock()
firstRecordIndex := v.firstRecordIndex
lastRecordIndex := v.lastRecordIndex
selectedRecordIndex := v.selectedRecordIndex
records := v.records
v.mu.RUnlock()
v.DrawForSubclass(screen, v)
x, y, width, height := v.GetInnerRect()
if height == 0 {
return
}
// No records in that bucket.
if firstRecordIndex == lastRecordIndex {
tview.Print(
screen, "Empty Bucket", x, y, width, tview.AlignCenter, tview.Styles.PrimaryTextColor,
)
return
}
for index := firstRecordIndex; index < lastRecordIndex; index++ {
result := records[index].Entry
text := result.String()
if index == selectedRecordIndex {
text = fmt.Sprintf("[:white]%s[:-]", text)
tview.Print(screen, text, x, y, width, tview.AlignLeft, tview.Styles.PrimitiveBackgroundColor)
} else {
tview.Print(screen, text, x, y, width, tview.AlignLeft, tview.Styles.PrimaryTextColor)
}
y++
}
}
func (v *RecordsView) InputHandler() func(event *tcell.EventKey, _ func(p tview.Primitive)) {
return v.WrapInputHandler(func(event *tcell.EventKey, _ func(p tview.Primitive)) {
switch m, k := event.Modifiers(), event.Key(); {
case m == 0 && k == tcell.KeyPgUp:
v.updateType = moveToPrevPage
case m == 0 && k == tcell.KeyPgDn:
v.updateType = moveToNextPage
case m == 0 && k == tcell.KeyUp:
v.updateType = moveUp
case m == 0 && k == tcell.KeyDown:
v.updateType = moveDown
case m == 0 && k == tcell.KeyHome:
v.updateType = moveHome
case m == 0 && k == tcell.KeyEnd:
v.updateType = moveEnd
case k == tcell.KeyEnter:
v.mu.RLock()
selectedRecordIndex := v.selectedRecordIndex
records := v.records
v.mu.RUnlock()
if len(records) != 0 {
current := records[selectedRecordIndex]
v.ui.moveNextPage(NewDetailedView(current.Entry.DetailedString()))
}
}
})
}

View file

@ -0,0 +1,18 @@
package tui
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
)
type Bucket struct {
Name []byte
Path [][]byte
Entry common.SchemaEntry
NextParser common.Parser
}
type Record struct {
Key, Value []byte
Path [][]byte
Entry common.SchemaEntry
}

View file

@ -0,0 +1,561 @@
package tui
import (
"context"
"errors"
"fmt"
"strings"
"sync/atomic"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common"
"github.com/davecgh/go-spew/spew"
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
"go.etcd.io/bbolt"
)
type Config struct {
LoadBufferSize int
SearchHistorySize int
LoadingIndicatorLag time.Duration
}
var DefaultConfig = Config{
LoadBufferSize: 100,
SearchHistorySize: 100,
LoadingIndicatorLag: 500 * time.Millisecond,
}
type Primitive interface {
tview.Primitive
Mount(ctx context.Context) error
Update(ctx context.Context) error
Unmount()
}
type UI struct {
*tview.Box
// Need to use context while updating pages those read data from a database.
// Context should be shared among all mount and updates. Current TUI library
// doesn't use contexts at all, so I do that feature by myself.
//nolint:containedctx
ctx context.Context
onStop func()
app *tview.Application
db *bbolt.DB
pageHistory []Primitive
mountedPage Primitive
pageToMount Primitive
pageStub tview.Primitive
infoBar *tview.TextView
searchBar *InputFieldWithHistory
loadingBar *LoadingBar
helpBar *tview.TextView
helpPage *HelpPage
searchErrorBar *tview.TextView
isSearching bool
isLoading atomic.Bool
isShowingError bool
isShowingHelp bool
loadBufferSize int
rootParser common.Parser
loadingIndicatorLag time.Duration
cancelLoading func()
filters map[string]func(string) (any, error)
compositeFilters map[string]func(string) (map[string]any, error)
filterHints map[string]string
}
func NewUI(
ctx context.Context,
app *tview.Application,
db *bbolt.DB,
rootParser common.Parser,
cfg *Config,
) *UI {
spew.Config.DisableMethods = true
if cfg == nil {
cfg = &DefaultConfig
}
ui := &UI{
Box: tview.NewBox(),
app: app,
db: db,
rootParser: rootParser,
filters: make(map[string]func(string) (any, error)),
compositeFilters: make(map[string]func(string) (map[string]any, error)),
filterHints: make(map[string]string),
loadBufferSize: cfg.LoadBufferSize,
loadingIndicatorLag: cfg.LoadingIndicatorLag,
}
ui.ctx, ui.onStop = context.WithCancel(ctx)
backgroundColor := ui.GetBackgroundColor()
textColor := tview.Styles.PrimaryTextColor
inverseBackgroundColor := textColor
inverseTextColor := backgroundColor
alertTextColor := tcell.ColorRed
ui.pageStub = tview.NewBox()
ui.infoBar = tview.NewTextView()
ui.infoBar.SetBackgroundColor(inverseBackgroundColor)
ui.infoBar.SetTextColor(inverseTextColor)
ui.infoBar.SetText(
fmt.Sprintf(" %s (press h for help, q to quit) ", db.Path()),
)
ui.searchBar = NewInputFieldWithHistory(cfg.SearchHistorySize)
ui.searchBar.SetFieldBackgroundColor(backgroundColor)
ui.searchBar.SetFieldTextColor(textColor)
ui.searchBar.SetLabelColor(textColor)
ui.searchBar.Focus(nil)
ui.searchBar.SetLabel("/")
ui.searchErrorBar = tview.NewTextView()
ui.searchErrorBar.SetBackgroundColor(backgroundColor)
ui.searchErrorBar.SetTextColor(alertTextColor)
ui.helpBar = tview.NewTextView()
ui.helpBar.SetBackgroundColor(inverseBackgroundColor)
ui.helpBar.SetTextColor(inverseTextColor)
ui.helpBar.SetText(" Press Enter for next page or Escape to exit help ")
ui.loadingBar = NewLoadingBar(ui.triggerDraw)
ui.pageToMount = NewBucketsView(ui, NewFilter(nil))
return ui
}
func (ui *UI) checkFilterExists(typ string) bool {
if _, ok := ui.filters[typ]; ok {
return true
}
if _, ok := ui.compositeFilters[typ]; ok {
return true
}
return false
}
func (ui *UI) AddFilter(
typ string,
parser func(string) (any, error),
helpHint string,
) error {
if ui.checkFilterExists(typ) {
return fmt.Errorf("filter %s already exists", typ)
}
ui.filters[typ] = parser
ui.filterHints[typ] = helpHint
return nil
}
func (ui *UI) AddCompositeFilter(
typ string,
parser func(string) (map[string]any, error),
helpHint string,
) error {
if ui.checkFilterExists(typ) {
return fmt.Errorf("filter %s already exists", typ)
}
ui.compositeFilters[typ] = parser
ui.filterHints[typ] = helpHint
return nil
}
func (ui *UI) stopOnError(err error) {
if err != nil {
ui.onStop()
ui.app.QueueEvent(tcell.NewEventError(err))
}
}
func (ui *UI) stop() {
ui.onStop()
ui.app.Stop()
}
func (ui *UI) movePrevPage() {
if len(ui.pageHistory) != 0 {
ui.mountedPage.Unmount()
ui.mountedPage = ui.pageHistory[len(ui.pageHistory)-1]
ui.pageHistory = ui.pageHistory[:len(ui.pageHistory)-1]
ui.triggerDraw()
}
}
func (ui *UI) moveNextPage(page Primitive) {
ui.pageToMount = page
ui.triggerDraw()
}
func (ui *UI) triggerDraw() {
go ui.app.QueueUpdateDraw(func() {})
}
func (ui *UI) Draw(screen tcell.Screen) {
if ui.isLoading.Load() {
ui.draw(screen)
return
}
ui.isLoading.Store(true)
ctx, cancel := context.WithCancel(ui.ctx)
ready := make(chan struct{})
go func() {
ui.load(ctx)
cancel()
close(ready)
ui.isLoading.Store(false)
}()
select {
case <-ready:
case <-time.After(ui.loadingIndicatorLag):
ui.loadingBar.Start(ui.ctx)
ui.cancelLoading = cancel
go func() {
<-ready
ui.loadingBar.Stop()
ui.triggerDraw()
}()
}
ui.draw(screen)
}
func (ui *UI) load(ctx context.Context) {
if ui.mountedPage == nil && ui.pageToMount == nil {
ui.stop()
return
}
if ui.pageToMount != nil {
ui.mountAndUpdate(ctx)
} else {
ui.update(ctx)
}
}
func (ui *UI) draw(screen tcell.Screen) {
ui.DrawForSubclass(screen, ui)
x, y, width, height := ui.GetInnerRect()
var (
pageToDraw tview.Primitive
barToDraw tview.Primitive
)
switch {
case ui.isShowingHelp:
if ui.helpPage == nil {
var filters []string
for f := range ui.filters {
filters = append(filters, f)
}
for f := range ui.compositeFilters {
filters = append(filters, f)
}
ui.helpPage = NewHelpPage(filters, ui.filterHints)
}
pageToDraw = ui.helpPage
case ui.mountedPage != nil:
pageToDraw = ui.mountedPage
default:
pageToDraw = ui.pageStub
}
pageToDraw.SetRect(x, y, width, height-1)
pageToDraw.Draw(screen)
// Search bar uses cursor and we need to hide it when another bar is drawn.
screen.HideCursor()
switch {
case ui.isLoading.Load():
barToDraw = ui.loadingBar
case ui.isSearching:
barToDraw = ui.searchBar
case ui.isShowingError:
barToDraw = ui.searchErrorBar
case ui.isShowingHelp:
barToDraw = ui.helpBar
default:
barToDraw = ui.infoBar
}
barToDraw.SetRect(x, y+height-1, width, 1)
barToDraw.Draw(screen)
}
func (ui *UI) mountAndUpdate(ctx context.Context) {
defer func() {
// Operation succeeded or was canceled, either way reset page to mount.
ui.pageToMount = nil
}()
// Mount should use app global context.
//nolint:contextcheck
err := ui.pageToMount.Mount(ui.ctx)
if err != nil {
ui.stopOnError(err)
return
}
x, y, width, height := ui.GetInnerRect()
ui.pageToMount.SetRect(x, y, width, height-1)
s := loadOp(ctx, ui.pageToMount.Update)
if s.err != nil {
ui.pageToMount.Unmount()
ui.stopOnError(s.err)
return
}
// Update was canceled.
if !s.done {
ui.pageToMount.Unmount()
return
}
if ui.mountedPage != nil {
ui.pageHistory = append(ui.pageHistory, ui.mountedPage)
}
ui.mountedPage = ui.pageToMount
}
func (ui *UI) update(ctx context.Context) {
x, y, width, height := ui.GetInnerRect()
ui.mountedPage.SetRect(x, y, width, height-1)
s := loadOp(ctx, ui.mountedPage.Update)
if s.err != nil {
ui.stopOnError(s.err)
return
}
}
type status struct {
done bool
err error
}
func loadOp(ctx context.Context, op func(ctx context.Context) error) status {
errCh := make(chan error)
go func() {
errCh <- op(ctx)
}()
select {
case <-ctx.Done():
return status{done: false, err: nil}
case err := <-errCh:
return status{done: true, err: err}
}
}
func (ui *UI) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) {
return ui.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) {
switch {
case ui.isLoading.Load():
ui.handleInputOnLoading(event)
case ui.isShowingHelp:
ui.handleInputOnShowingHelp(event)
case ui.isShowingError:
ui.handleInputOnShowingError()
case ui.isSearching:
ui.handleInputOnSearching(event)
default:
ui.handleInput(event)
}
})
}
func (ui *UI) handleInput(event *tcell.EventKey) {
m, k, r := event.Modifiers(), event.Key(), event.Rune()
switch {
case k == tcell.KeyEsc:
ui.movePrevPage()
case m == 0 && k == tcell.KeyRune && r == 'h':
ui.isShowingHelp = true
case m == 0 && k == tcell.KeyRune && r == '/':
ui.isSearching = true
case m == 0 && k == tcell.KeyRune && r == 'q':
ui.stop()
default:
if ui.mountedPage != nil {
ui.mountedPage.InputHandler()(event, func(tview.Primitive) {})
}
}
}
func (ui *UI) handleInputOnLoading(event *tcell.EventKey) {
switch k, r := event.Key(), event.Rune(); {
case k == tcell.KeyEsc:
ui.cancelLoading()
case k == tcell.KeyRune && r == 'q':
ui.stop()
}
}
func (ui *UI) handleInputOnShowingError() {
ui.isShowingError = false
ui.isSearching = true
}
func (ui *UI) handleInputOnShowingHelp(event *tcell.EventKey) {
k, r := event.Key(), event.Rune()
switch {
case k == tcell.KeyEsc:
ui.isShowingHelp = false
case k == tcell.KeyRune && r == 'q':
ui.stop()
default:
ui.helpPage.InputHandler()(event, func(tview.Primitive) {})
}
}
func (ui *UI) handleInputOnSearching(event *tcell.EventKey) {
m, k := event.Modifiers(), event.Key()
switch {
case k == tcell.KeyEnter:
prompt := ui.searchBar.GetText()
res, err := ui.processPrompt(prompt)
if err != nil {
ui.isShowingError = true
ui.isSearching = false
ui.searchErrorBar.SetText(err.Error() + " (press any key to continue)")
return
}
switch ui.mountedPage.(type) {
case *BucketsView:
ui.moveNextPage(NewBucketsView(ui, res))
case *RecordsView:
bucket := ui.mountedPage.(*RecordsView).bucket
ui.moveNextPage(NewRecordsView(ui, bucket, res))
}
if ui.searchBar.GetText() != "" {
ui.searchBar.AddToHistory(ui.searchBar.GetText())
}
ui.searchBar.SetText("")
ui.isSearching = false
case k == tcell.KeyEsc:
ui.isSearching = false
case (k == tcell.KeyBackspace2 || m&tcell.ModCtrl != 0 && k == tcell.KeyETB) && len(ui.searchBar.GetText()) == 0:
ui.isSearching = false
default:
ui.searchBar.InputHandler()(event, func(tview.Primitive) {})
}
ui.Box.MouseHandler()
}
func (ui *UI) WithPrompt(prompt string) error {
filter, err := ui.processPrompt(prompt)
if err != nil {
return err
}
ui.pageToMount = NewBucketsView(ui, filter)
if prompt != "" {
ui.searchBar.AddToHistory(prompt)
}
return nil
}
func (ui *UI) processPrompt(prompt string) (filter *Filter, err error) {
if prompt == "" {
return NewFilter(nil), nil
}
filterMap := make(map[string]any)
for _, filterString := range strings.Split(prompt, "+") {
parts := strings.Split(filterString, ":")
if len(parts) != 2 {
return nil, errors.New("expected 'tag:value [+ tag:value]...'")
}
filterTag := strings.TrimSpace(parts[0])
filterValueString := strings.TrimSpace(parts[1])
if _, exists := filterMap[filterTag]; exists {
return nil, fmt.Errorf("duplicate filter tag '%s'", filterTag)
}
parser, ok := ui.filters[filterTag]
if ok {
filterValue, err := parser(filterValueString)
if err != nil {
return nil, fmt.Errorf("can't parse '%s' filter value: %w", filterTag, err)
}
filterMap[filterTag] = filterValue
continue
}
compositeParser, ok := ui.compositeFilters[filterTag]
if ok {
compositeFilterValue, err := compositeParser(filterValueString)
if err != nil {
return nil, fmt.Errorf(
"can't parse '%s' filter value '%s': %w",
filterTag, filterValueString, err,
)
}
for tag, value := range compositeFilterValue {
if _, exists := filterMap[tag]; exists {
return nil, fmt.Errorf(
"found duplicate filter tag '%s' while processing composite filter with tag '%s'",
tag, filterTag,
)
}
filterMap[tag] = value
}
continue
}
return nil, fmt.Errorf("unknown filter tag '%s'", filterTag)
}
return NewFilter(filterMap), nil
}

View file

@ -0,0 +1,97 @@
package tui
import (
"errors"
"strings"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/mr-tron/base58"
)
func CIDParser(s string) (any, error) {
data, err := base58.Decode(s)
if err != nil {
return nil, err
}
var id cid.ID
if err = id.Decode(data); err != nil {
return nil, err
}
return id, nil
}
func OIDParser(s string) (any, error) {
data, err := base58.Decode(s)
if err != nil {
return nil, err
}
var id oid.ID
if err = id.Decode(data); err != nil {
return nil, err
}
return id, nil
}
func AddressParser(s string) (map[string]any, error) {
m := make(map[string]any)
parts := strings.Split(s, "/")
if len(parts) != 2 {
return nil, errors.New("expected <cid>/<oid>")
}
cnr, err := CIDParser(parts[0])
if err != nil {
return nil, err
}
obj, err := OIDParser(parts[1])
if err != nil {
return nil, err
}
m["cid"] = cnr
m["oid"] = obj
return m, nil
}
func keyParser(s string) (any, error) {
if s == "" {
return nil, errors.New("empty attribute key")
}
return s, nil
}
func valueParser(s string) (any, error) {
if s == "" {
return nil, errors.New("empty attribute value")
}
return s, nil
}
func AttributeParser(s string) (map[string]any, error) {
m := make(map[string]any)
parts := strings.Split(s, "/")
if len(parts) != 1 && len(parts) != 2 {
return nil, errors.New("expected <key> or <key>/<value>")
}
key, err := keyParser(parts[0])
if err != nil {
return nil, err
}
m["key"] = key
if len(parts) == 1 {
return m, nil
}
value, err := valueParser(parts[1])
if err != nil {
return nil, err
}
m["value"] = value
return m, nil
}

View file

@ -17,5 +17,5 @@ var Root = &cobra.Command{
}
func init() {
Root.AddCommand(listCMD, inspectCMD)
Root.AddCommand(listCMD, inspectCMD, tuiCMD)
}

View file

@ -0,0 +1,79 @@
package writecache
import (
"context"
"fmt"
common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal"
schema "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/writecache"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui"
"github.com/rivo/tview"
"github.com/spf13/cobra"
"go.etcd.io/bbolt"
)
var tuiCMD = &cobra.Command{
Use: "explore",
Short: "Write cache exploration with a terminal UI",
Long: `Launch a terminal UI to explore write cache and search for data.
Available search filters:
- cid CID
- oid OID
- addr CID/OID
`,
Run: tuiFunc,
}
var initialPrompt string
func init() {
common.AddComponentPathFlag(tuiCMD, &vPath)
tuiCMD.Flags().StringVar(
&initialPrompt,
"filter",
"",
"Filter prompt to start with, format 'tag:value [+ tag:value]...'",
)
}
func tuiFunc(cmd *cobra.Command, _ []string) {
common.ExitOnErr(cmd, runTUI(cmd))
}
func runTUI(cmd *cobra.Command) error {
db, err := openDB(false)
if err != nil {
return fmt.Errorf("couldn't open database: %w", err)
}
defer db.Close()
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
app := tview.NewApplication()
ui := tui.NewUI(ctx, app, db, schema.WritecacheParser, nil)
_ = ui.AddFilter("cid", tui.CIDParser, "CID")
_ = ui.AddFilter("oid", tui.OIDParser, "OID")
_ = ui.AddCompositeFilter("addr", tui.AddressParser, "CID/OID")
err = ui.WithPrompt(initialPrompt)
if err != nil {
return fmt.Errorf("invalid filter prompt: %w", err)
}
app.SetRoot(ui, true).SetFocus(ui)
return app.Run()
}
func openDB(writable bool) (*bbolt.DB, error) {
db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{
ReadOnly: !writable,
})
if err != nil {
return nil, err
}
return db, nil
}

8
go.mod
View file

@ -17,7 +17,9 @@ require (
github.com/VictoriaMetrics/easyproto v0.1.4
github.com/cheggaaa/pb v1.0.29
github.com/chzyer/readline v1.5.1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
github.com/gdamore/tcell/v2 v2.7.4
github.com/go-pkgz/expirable-cache/v3 v3.0.0
github.com/google/uuid v1.6.0
github.com/hashicorp/golang-lru/v2 v2.0.7
@ -30,6 +32,7 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/panjf2000/ants/v2 v2.9.0
github.com/prometheus/client_golang v1.19.0
github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130
github.com/spf13/cast v1.6.0
github.com/spf13/cobra v1.8.1
github.com/spf13/pflag v1.0.5
@ -65,10 +68,10 @@ require (
github.com/consensys/bavard v0.1.13 // indirect
github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/davidmz/go-pageant v1.0.2 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/go-fed/httpsig v1.1.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
@ -85,6 +88,7 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/klauspost/reedsolomon v1.12.1 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
@ -103,7 +107,7 @@ require (
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.11.0 // indirect

BIN
go.sum

Binary file not shown.

View file

@ -0,0 +1,132 @@
package internal
import (
"crypto/sha256"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
"git.frostfs.info/TrueCloudLab/tzhash/tz"
"golang.org/x/exp/rand"
)
func GeneratePayloadPool(count uint, size uint) [][]byte {
pool := [][]byte{}
for i := uint(0); i < count; i++ {
payload := make([]byte, size)
_, _ = rand.Read(payload)
pool = append(pool, payload)
}
return pool
}
func GenerateAttributePool(count uint) []objectSDK.Attribute {
pool := []objectSDK.Attribute{}
for i := uint(0); i < count; i++ {
for j := uint(0); j < count; j++ {
attr := *objectSDK.NewAttribute()
attr.SetKey(fmt.Sprintf("key%d", i))
attr.SetValue(fmt.Sprintf("value%d", j))
pool = append(pool, attr)
}
}
return pool
}
func GenerateOwnerPool(count uint) []user.ID {
pool := []user.ID{}
for i := uint(0); i < count; i++ {
pool = append(pool, usertest.ID())
}
return pool
}
type ObjectOption func(obj *objectSDK.Object)
func GenerateObject(options ...ObjectOption) *objectSDK.Object {
var ver version.Version
ver.SetMajor(2)
ver.SetMinor(1)
payload := make([]byte, 0)
var csum checksum.Checksum
csum.SetSHA256(sha256.Sum256(payload))
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
obj := objectSDK.New()
obj.SetID(oidtest.ID())
obj.SetOwnerID(usertest.ID())
obj.SetContainerID(cidtest.ID())
header := objecttest.Object().GetECHeader()
header.SetParent(oidtest.ID())
obj.SetECHeader(header)
obj.SetVersion(&ver)
obj.SetPayload(payload)
obj.SetPayloadSize(uint64(len(payload)))
obj.SetPayloadChecksum(csum)
obj.SetPayloadHomomorphicHash(csumTZ)
for _, option := range options {
option(obj)
}
return obj
}
func WithContainerID(cid cid.ID) ObjectOption {
return func(obj *objectSDK.Object) {
obj.SetContainerID(cid)
}
}
func WithType(typ objectSDK.Type) ObjectOption {
return func(obj *objectSDK.Object) {
obj.SetType(typ)
}
}
func WithPayloadFromPool(pool [][]byte) ObjectOption {
payload := pool[rand.Intn(len(pool))]
var csum checksum.Checksum
csum.SetSHA256(sha256.Sum256(payload))
var csumTZ checksum.Checksum
csumTZ.SetTillichZemor(tz.Sum(csum.Value()))
return func(obj *objectSDK.Object) {
obj.SetPayload(payload)
obj.SetPayloadSize(uint64(len(payload)))
obj.SetPayloadChecksum(csum)
obj.SetPayloadHomomorphicHash(csumTZ)
}
}
func WithAttributesFromPool(pool []objectSDK.Attribute, count uint) ObjectOption {
return func(obj *objectSDK.Object) {
attrs := []objectSDK.Attribute{}
for i := uint(0); i < count; i++ {
attrs = append(attrs, pool[rand.Intn(len(pool))])
}
obj.SetAttributes(attrs...)
}
}
func WithOwnerIDFromPool(pool []user.ID) ObjectOption {
return func(obj *objectSDK.Object) {
obj.SetOwnerID(pool[rand.Intn(len(pool))])
}
}

View file

@ -0,0 +1,263 @@
package internal
import (
"context"
"fmt"
"math/rand"
"sync"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"golang.org/x/sync/errgroup"
)
type EpochState struct{}
func (s EpochState) CurrentEpoch() uint64 {
return 0
}
func PopulateWithObjects(
ctx context.Context,
db *meta.DB,
group *errgroup.Group,
count uint,
factory func() *objectSDK.Object,
) {
digits := "0123456789"
for i := uint(0); i < count; i++ {
obj := factory()
id := []byte(fmt.Sprintf(
"%c/%c/%c",
digits[rand.Int()%len(digits)],
digits[rand.Int()%len(digits)],
digits[rand.Int()%len(digits)],
))
prm := meta.PutPrm{}
prm.SetObject(obj)
prm.SetStorageID(id)
group.Go(func() error {
if _, err := db.Put(ctx, prm); err != nil {
return fmt.Errorf("couldn't put an object: %w", err)
}
return nil
})
}
}
func PopulateWithBigObjects(
ctx context.Context,
db *meta.DB,
group *errgroup.Group,
count uint,
factory func() *objectSDK.Object,
) {
for i := uint(0); i < count; i++ {
group.Go(func() error {
if err := populateWithBigObject(ctx, db, factory); err != nil {
return fmt.Errorf("couldn't put a big object: %w", err)
}
return nil
})
}
}
func populateWithBigObject(
ctx context.Context,
db *meta.DB,
factory func() *objectSDK.Object,
) error {
t := &target{db: db}
pk, _ := keys.NewPrivateKey()
p := transformer.NewPayloadSizeLimiter(transformer.Params{
Key: &pk.PrivateKey,
NextTargetInit: func() transformer.ObjectWriter { return t },
NetworkState: EpochState{},
MaxSize: 10,
})
obj := factory()
payload := make([]byte, 30)
err := p.WriteHeader(ctx, obj)
if err != nil {
return err
}
_, err = p.Write(ctx, payload)
if err != nil {
return err
}
_, err = p.Close(ctx)
if err != nil {
return err
}
return nil
}
type target struct {
db *meta.DB
}
func (t *target) WriteObject(ctx context.Context, obj *objectSDK.Object) error {
prm := meta.PutPrm{}
prm.SetObject(obj)
_, err := t.db.Put(ctx, prm)
return err
}
func PopulateGraveyard(
ctx context.Context,
db *meta.DB,
group *errgroup.Group,
workBufferSize int,
count uint,
factory func() *objectSDK.Object,
) {
ts := factory()
ts.SetType(objectSDK.TypeTombstone)
prm := meta.PutPrm{}
prm.SetObject(ts)
group.Go(func() error {
if _, err := db.Put(ctx, prm); err != nil {
return fmt.Errorf("couldn't put a tombstone object: %w", err)
}
return nil
})
cID, _ := ts.ContainerID()
oID, _ := ts.ID()
var tsAddr oid.Address
tsAddr.SetContainer(cID)
tsAddr.SetObject(oID)
addrs := make(chan oid.Address, workBufferSize)
go func() {
defer close(addrs)
wg := &sync.WaitGroup{}
wg.Add(int(count))
for i := uint(0); i < count; i++ {
obj := factory()
prm := meta.PutPrm{}
prm.SetObject(obj)
group.Go(func() error {
defer wg.Done()
if _, err := db.Put(ctx, prm); err != nil {
return fmt.Errorf("couldn't put an object: %w", err)
}
cID, _ := obj.ContainerID()
oID, _ := obj.ID()
var addr oid.Address
addr.SetContainer(cID)
addr.SetObject(oID)
addrs <- addr
return nil
})
}
wg.Wait()
}()
go func() {
for addr := range addrs {
prm := meta.InhumePrm{}
prm.SetAddresses(addr)
prm.SetTombstoneAddress(tsAddr)
group.Go(func() error {
if _, err := db.Inhume(ctx, prm); err != nil {
return fmt.Errorf("couldn't inhume an object: %w", err)
}
return nil
})
}
}()
}
func PopulateLocked(
ctx context.Context,
db *meta.DB,
group *errgroup.Group,
workBufferSize int,
count uint,
factory func() *objectSDK.Object,
) {
locker := factory()
locker.SetType(objectSDK.TypeLock)
prm := meta.PutPrm{}
prm.SetObject(locker)
group.Go(func() error {
if _, err := db.Put(ctx, prm); err != nil {
return fmt.Errorf("couldn't put a locker object: %w", err)
}
return nil
})
ids := make(chan oid.ID, workBufferSize)
go func() {
defer close(ids)
wg := &sync.WaitGroup{}
wg.Add(int(count))
for i := uint(0); i < count; i++ {
defer wg.Done()
obj := factory()
prm := meta.PutPrm{}
prm.SetObject(obj)
group.Go(func() error {
if _, err := db.Put(ctx, prm); err != nil {
return fmt.Errorf("couldn't put an object: %w", err)
}
id, _ := obj.ID()
ids <- id
return nil
})
}
wg.Wait()
}()
go func() {
for id := range ids {
lockerCID, _ := locker.ContainerID()
lockerOID, _ := locker.ID()
group.Go(func() error {
if err := db.Lock(ctx, lockerCID, lockerOID, []oid.ID{id}); err != nil {
return fmt.Errorf("couldn't lock an object: %w", err)
}
return nil
})
}
}()
}

View file

@ -0,0 +1,159 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"os"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/scripts/populate-metabase/internal"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
"golang.org/x/sync/errgroup"
)
var (
path string
force bool
jobs uint
numContainers,
numObjects,
numAttributesPerObj,
numOwners,
numPayloads,
numAttributes uint
)
func main() {
flag.StringVar(&path, "path", "", "Path to metabase")
flag.BoolVar(&force, "force", false, "Rewrite existing database")
flag.UintVar(&jobs, "j", 10000, "Number of jobs to run")
flag.UintVar(&numContainers, "containers", 0, "Number of containers to be created")
flag.UintVar(&numObjects, "objects", 0, "Number of objects per container")
flag.UintVar(&numAttributesPerObj, "attributes", 0, "Number of attributes per object")
flag.UintVar(&numOwners, "distinct-owners", 10, "Number of distinct owners to be used")
flag.UintVar(&numPayloads, "distinct-payloads", 10, "Number of distinct payloads to be used")
aarifullin marked this conversation as resolved Outdated

If you introduce exitIfFalse (instead exitOnZero), then you also can use this one-line manner to exit the programm on error :)

exitIfFalse("must have payloads", *numPayloads > 0)
exitIfFalse("must have attributes", *numAttributes > 0)
exitIfFalse("must have owners", *numOwners > 0)
exitIfFalse("path to metabase not specified", len(*path) > 0)
exitIfFalse("object can't have more attributes than available", *numAttributesPerObj <= *numAttributes)

WDYT?

If you introduce `exitIfFalse` (instead `exitOnZero`), then you also can use this one-line manner to exit the programm on error :) ```go exitIfFalse("must have payloads", *numPayloads > 0) exitIfFalse("must have attributes", *numAttributes > 0) exitIfFalse("must have owners", *numOwners > 0) exitIfFalse("path to metabase not specified", len(*path) > 0) exitIfFalse("object can't have more attributes than available", *numAttributesPerObj <= *numAttributes) ``` WDYT?

I think it's awesome. I've changed it.

I think it's awesome. I've changed it.
flag.UintVar(&numAttributes, "distinct-attributes", 10, "Number of distinct attributes to be used")
flag.Parse()
exitIf(numPayloads == 0, "must have payloads\n")
exitIf(numAttributes == 0, "must have attributes\n")
exitIf(numOwners == 0, "must have owners\n")
exitIf(len(path) == 0, "path to metabase not specified\n")
exitIf(
numAttributesPerObj > numAttributes,
"object can't have more attributes than available\n",
)
info, err := os.Stat(path)
exitIf(
err != nil && !errors.Is(err, os.ErrNotExist),
"couldn't get path info: %s\n", err,
)
// Path exits.
if err == nil {
exitIf(info.IsDir(), "path is a directory\n")
exitIf(!force, "couldn't rewrite existing file, use '-force' flag\n")
err = os.Remove(path)
exitIf(err != nil, "couldn't remove existing file: %s\n", err)
}
err = populate()
exitIf(err != nil, "couldn't populate the metabase: %s\n", err)
}
func getObjectFactory(opts ...internal.ObjectOption) func() *objectSDK.Object {
return func() *objectSDK.Object {
return internal.GenerateObject(opts...)
}
}
func populate() (err error) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
db := meta.New([]meta.Option{
meta.WithPath(path),
meta.WithPermissions(0o600),
meta.WithEpochState(internal.EpochState{}),
}...)
if err = db.Open(ctx, mode.ReadWrite); err != nil {
return fmt.Errorf("couldn't open the metabase: %w", err)
}
defer func() {
if errOnClose := db.Close(); errOnClose != nil {
err = errors.Join(
err,
fmt.Errorf("couldn't close the metabase: %w", db.Close()),
)
}
}()
if err = db.Init(); err != nil {
return fmt.Errorf("couldn't init the metabase: %w", err)
}
payloads := internal.GeneratePayloadPool(numPayloads, 32)
attributes := internal.GenerateAttributePool(numAttributes)
owners := internal.GenerateOwnerPool(numOwners)
types := []objectSDK.Type{
objectSDK.TypeRegular,
objectSDK.TypeLock,
objectSDK.TypeTombstone,
}
eg, ctx := errgroup.WithContext(ctx)
eg.SetLimit(int(jobs))
for i := uint(0); i < numContainers; i++ {
cid := cidtest.ID()
for _, typ := range types {
internal.PopulateWithObjects(ctx, db, eg, numObjects, getObjectFactory(
internal.WithContainerID(cid),
internal.WithType(typ),
internal.WithPayloadFromPool(payloads),
internal.WithOwnerIDFromPool(owners),
internal.WithAttributesFromPool(attributes, numAttributesPerObj),
))
}
internal.PopulateWithBigObjects(ctx, db, eg, numObjects, getObjectFactory(
internal.WithContainerID(cid),
internal.WithType(objectSDK.TypeRegular),
internal.WithAttributesFromPool(attributes, numAttributesPerObj),
internal.WithOwnerIDFromPool(owners),
))
internal.PopulateGraveyard(ctx, db, eg, int(jobs), numObjects, getObjectFactory(
internal.WithContainerID(cid),
internal.WithType(objectSDK.TypeRegular),
internal.WithAttributesFromPool(attributes, numAttributesPerObj),
internal.WithOwnerIDFromPool(owners),
))
internal.PopulateLocked(ctx, db, eg, int(jobs), numObjects, getObjectFactory(
internal.WithContainerID(cid),
internal.WithType(objectSDK.TypeRegular),
internal.WithAttributesFromPool(attributes, numAttributesPerObj),
internal.WithOwnerIDFromPool(owners),
))
}
return eg.Wait()
}
func exitIf(cond bool, format string, args ...any) {
if cond {
fmt.Fprintf(os.Stderr, format, args...)
os.Exit(1)
}
}