From 7525fbd9a9294bd847dc8ab2edc3b08f26696c94 Mon Sep 17 00:00:00 2001 From: Aleksey Kravchenko Date: Mon, 17 Feb 2025 20:55:30 +0300 Subject: [PATCH] [#13] Add info FrostFS backend cmd Signed-off-by: Aleksey Kravchenko --- backend/frostfs/frostfs.go | 322 +++++++++++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) diff --git a/backend/frostfs/frostfs.go b/backend/frostfs/frostfs.go index 8d0d8566b..fb8348201 100644 --- a/backend/frostfs/frostfs.go +++ b/backend/frostfs/frostfs.go @@ -17,6 +17,8 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" sdkClient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -41,6 +43,7 @@ func init() { Name: "frostfs", Description: "Distributed, decentralized object storage FrostFS", NewFs: NewFs, + CommandHelp: commandHelp, Options: []fs.Option{ { Name: "endpoint", @@ -156,6 +159,59 @@ func init() { }) } +var commandHelp = []fs.CommandHelp{ + { + Name: "info", + Short: "Show information about the FrostFS objects and containers", + Long: `This command can be used to get information about the FrostFS objects and containers. + +Usage Examples: + + rclone backend info frostfs:container/path/to/dir + rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt + rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt -o "format={cid}:{oid}" + +The optional "format" flag overrides the information output. In this example, if an object is stored in +a container with the identifier "9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1" and +its own identifier is "4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP", the output of this command will be +"9mvN7hsucoGoHjxPqrWmDipnMaemGVDqrxxPyynn1:4VpcNFsZqsQt1Gnfw2utBnzn5Blgc7i4kvtuXyKyJp". + +The default output format is the same as that of the frostfs-cli utility, +with the "container get" and "object head" options. Here is an example of output: + + --- Container info --- + CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 + Owner ID: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX + Created: 2025-02-17 15:07:51 +0300 MSK + Attributes: + Timestamp=1739794071 + Name=test + __SYSTEM__NAME=test + __SYSTEM__ZONE=container + __SYSTEM__DISABLE_HOMOMORPHIC_HASHING=true + Placement policy: + REP 3 + + --- Object info --- + ID: 4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP + CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 + Owner: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX + CreatedAt: 559 + Size: 402905 + HomoHash: + Checksum: 2a068fe24c53bc8bf7d6bbb997414f7938b080305dc45f9fd3ff684bc11fbb7b + Type: REGULAR + Attributes: + FileName=cat.png + FilePath=/dir1/dir2/dir3/cat.png + Timestamp=1733410524 (2024-12-05 17:55:24 +0300 MSK) + ID signature: + public key: 026b7c7a7a16225eb13a5a733495a1bcdd1f016dfa9193498821379b0de2ba6870 + signature: 049f6712c8378d323269b605a282bcacd7565ce2eefe1f10a9739c48945f739d95102c478b9cb1d429cd3330b4b5262e725392e322de3bbfa4ce18a9c842289219 +`, + }, +} + var errMalformedObject = errors.New("malformed object") // Options defines the configuration for this backend @@ -213,6 +269,272 @@ type Object struct { timestamp time.Time } +// Command the backend to run a named command +// +// The command run is name +// args may be used to read arguments from +// opts may be used to read optional arguments from +func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) { + switch name { + case "info": + return f.infoCmd(ctx, arg, opt) + default: + return nil, fs.ErrorCommandNotFound + } +} + +func (f *Fs) containerInfo(ctx context.Context, cnrID cid.ID) (container.Container, error) { + prm := pool.PrmContainerGet{ + ContainerID: cnrID, + } + var cnr container.Container + cnr, err := f.pool.GetContainer(ctx, prm) + if err != nil { + return container.Container{}, fmt.Errorf("couldn't get container '%s': %w", cnrID, err) + } + return cnr, err +} + +func (f *Fs) getObjectsHead(ctx context.Context, cnrID cid.ID, objIDs []oid.ID) ([]object.Object, error) { + var res []object.Object + for _, objID := range objIDs { + var prmHead pool.PrmObjectHead + prmHead.SetAddress(newAddress(cnrID, objID)) + + obj, err := f.pool.HeadObject(ctx, prmHead) + if err != nil { + return nil, err + } + res = append(res, obj) + } + + return res, nil +} + +type printer struct { + w io.Writer + err error +} + +func newPrinter(w io.Writer) *printer { + return &printer{w: w} +} + +func (p *printer) printf(format string, a ...interface{}) { + if p.err != nil { + return + } + if _, err := fmt.Fprintf(p.w, format, a...); err != nil { + p.err = err + } +} + +func (p *printer) lastError() error { + return p.err +} + +func (p *printer) printContainerInfo(cnrID cid.ID, cnr container.Container) { + p.printf("CID: %v\nOwner ID: %v", cnrID, cnr.Owner()) + var timestamp time.Time + var attrs []string + cnr.IterateAttributes(func(key string, value string) { + attrs = append(attrs, fmt.Sprintf(" %v=%v", key, value)) + if key == object.AttributeTimestamp { + val, err := strconv.ParseInt(value, 10, 64) + if err == nil { + timestamp = time.Unix(val, 0) + } + } + }) + if !timestamp.IsZero() { + p.printf("\nCreated: %v", timestamp) + } + if len(attrs) > 0 { + p.printf("\nAttributes:\n%s", strings.Join(attrs, "\n")) + } + + s := bytes.NewBufferString("") + if err := cnr.PlacementPolicy().WriteStringTo(s); err != nil { + return + } + + p.printf("\nPlacement policy:\n%s", s.String()) +} + +func (p *printer) printChecksum(name string, recv func() (checksum.Checksum, bool)) { + var strVal string + + cs, csSet := recv() + if csSet { + strVal = hex.EncodeToString(cs.Value()) + } else { + strVal = "" + } + + p.printf("\n%s: %s", name, strVal) +} + +func (p *printer) printObject(obj *object.Object) { + objIDStr := "" + cnrIDStr := objIDStr + if objID, ok := obj.ID(); ok { + objIDStr = objID.String() + } + if cnrID, ok := obj.ContainerID(); ok { + cnrIDStr = cnrID.String() + } + p.printf("\nID: %v", objIDStr) + p.printf("\nCID: %v", cnrIDStr) + p.printf("\nOwner: %s", obj.OwnerID()) + p.printf("\nCreatedAt: %d", obj.CreationEpoch()) + p.printf("\nSize: %d", obj.PayloadSize()) + p.printChecksum("HomoHash", obj.PayloadHomomorphicHash) + p.printChecksum("Checksum", obj.PayloadChecksum) + p.printf("\nType: %s", obj.Type()) + + p.printf("\nAttributes:") + for _, attr := range obj.Attributes() { + if attr.Key() == object.AttributeTimestamp { + var strVal string + val, err := strconv.ParseInt(attr.Value(), 10, 64) + if err == nil { + strVal = time.Unix(val, 0).String() + } else { + strVal = "malformed" + } + p.printf("\n %s=%s (%s)", + attr.Key(), + attr.Value(), + strVal) + continue + } + p.printf("\n %s=%s", attr.Key(), attr.Value()) + } + + if signature := obj.Signature(); signature != nil { + p.printf("\nID signature:") + + var sigV2 refs.Signature + signature.WriteToV2(&sigV2) + + p.printf("\n public key: %s", hex.EncodeToString(sigV2.GetKey())) + p.printf("\n signature: %s", hex.EncodeToString(sigV2.GetSign())) + } + + if ecHeader := obj.ECHeader(); ecHeader != nil { + p.printf("\nEC header:") + + p.printf("\n parent object ID: %s", ecHeader.Parent().EncodeToString()) + p.printf("\n index: %d", ecHeader.Index()) + p.printf("\n total: %d", ecHeader.Total()) + p.printf("\n header length: %d", ecHeader.HeaderLength()) + } + + p.printSplitHeader(obj) +} + +func (p *printer) printSplitHeader(obj *object.Object) { + if splitID := obj.SplitID(); splitID != nil { + p.printf("Split ID: %s\n", splitID) + } + + if objID, ok := obj.ParentID(); ok { + p.printf("Split ParentID: %s\n", objID) + } + + if prev, ok := obj.PreviousID(); ok { + p.printf("\nSplit PreviousID: %s", prev) + } + + for _, child := range obj.Children() { + p.printf("\nSplit ChildID: %s", child.String()) + } + + parent := obj.Parent() + if parent != nil { + p.printf("\n\nSplit Parent Header:") + + p.printObject(parent) + } + +} + +func formattedInfoOutput(format string, cnrID cid.ID, objHeads []object.Object) (string, error) { + format = strings.ReplaceAll(format, "{cid}", cnrID.String()) + objIDStr := "" + if len(objHeads) > 0 { + objID, ok := objHeads[0].ID() + if ok { + objIDStr = objID.String() + } + } + + return strings.ReplaceAll(format, "{oid}", objIDStr), nil +} + +func (f *Fs) infoCmd(ctx context.Context, arg []string, opt map[string]string) (out interface{}, err error) { + var cnrID cid.ID + + if cnrID, err = f.resolveContainerID(ctx, f.rootContainer); err != nil { + return nil, err + } + + var format string + for k, v := range opt { + switch k { + case "format": + format = v + default: + return nil, fmt.Errorf("unknown option \"%s\"", k) + } + } + + var objIDs []oid.ID + var filePath string + if len(arg) > 0 { + filePath = strings.TrimPrefix(arg[0], "/") + if f.rootDirectory != "" { + filePath = f.rootDirectory + "/" + filePath + } + + if objIDs, err = f.findObjectsFilePath(ctx, cnrID, filePath); err != nil { + return + } + } + + cnr, err := f.containerInfo(ctx, cnrID) + if err != nil { + return + } + var objHeads []object.Object + if objHeads, err = f.getObjectsHead(ctx, cnrID, objIDs); err != nil { + return + } + + if format != "" { + return formattedInfoOutput(format, cnrID, objHeads) + } + + w := bytes.NewBufferString("") + p := newPrinter(w) + p.printf(" --- Container info ---\n") + p.printContainerInfo(cnrID, cnr) + + if len(arg) > 0 { + p.printf("\n\n --- Object info ---") + if len(objHeads) > 0 { + // Print info about the first object only + p.printObject(&objHeads[0]) + } else { + p.printf("\nNo object with \"%s\" file path was found", filePath) + } + } + if err := p.lastError(); err != nil { + return nil, err + } + return w.String(), nil +} + // Shutdown the backend, closing any background tasks and any // cached connections. func (f *Fs) Shutdown(_ context.Context) error {