From 7525fbd9a9294bd847dc8ab2edc3b08f26696c94 Mon Sep 17 00:00:00 2001 From: Aleksey Kravchenko Date: Mon, 17 Feb 2025 20:55:30 +0300 Subject: [PATCH 1/2] [#13] Add info FrostFS backend cmd Signed-off-by: Aleksey Kravchenko --- backend/frostfs/frostfs.go | 322 +++++++++++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) diff --git a/backend/frostfs/frostfs.go b/backend/frostfs/frostfs.go index 8d0d8566b..fb8348201 100644 --- a/backend/frostfs/frostfs.go +++ b/backend/frostfs/frostfs.go @@ -17,6 +17,8 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" sdkClient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -41,6 +43,7 @@ func init() { Name: "frostfs", Description: "Distributed, decentralized object storage FrostFS", NewFs: NewFs, + CommandHelp: commandHelp, Options: []fs.Option{ { Name: "endpoint", @@ -156,6 +159,59 @@ func init() { }) } +var commandHelp = []fs.CommandHelp{ + { + Name: "info", + Short: "Show information about the FrostFS objects and containers", + Long: `This command can be used to get information about the FrostFS objects and containers. + +Usage Examples: + + rclone backend info frostfs:container/path/to/dir + rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt + rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt -o "format={cid}:{oid}" + +The optional "format" flag overrides the information output. In this example, if an object is stored in +a container with the identifier "9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1" and +its own identifier is "4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP", the output of this command will be +"9mvN7hsucoGoHjxPqrWmDipnMaemGVDqrxxPyynn1:4VpcNFsZqsQt1Gnfw2utBnzn5Blgc7i4kvtuXyKyJp". + +The default output format is the same as that of the frostfs-cli utility, +with the "container get" and "object head" options. Here is an example of output: + + --- Container info --- + CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 + Owner ID: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX + Created: 2025-02-17 15:07:51 +0300 MSK + Attributes: + Timestamp=1739794071 + Name=test + __SYSTEM__NAME=test + __SYSTEM__ZONE=container + __SYSTEM__DISABLE_HOMOMORPHIC_HASHING=true + Placement policy: + REP 3 + + --- Object info --- + ID: 4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP + CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 + Owner: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX + CreatedAt: 559 + Size: 402905 + HomoHash: + Checksum: 2a068fe24c53bc8bf7d6bbb997414f7938b080305dc45f9fd3ff684bc11fbb7b + Type: REGULAR + Attributes: + FileName=cat.png + FilePath=/dir1/dir2/dir3/cat.png + Timestamp=1733410524 (2024-12-05 17:55:24 +0300 MSK) + ID signature: + public key: 026b7c7a7a16225eb13a5a733495a1bcdd1f016dfa9193498821379b0de2ba6870 + signature: 049f6712c8378d323269b605a282bcacd7565ce2eefe1f10a9739c48945f739d95102c478b9cb1d429cd3330b4b5262e725392e322de3bbfa4ce18a9c842289219 +`, + }, +} + var errMalformedObject = errors.New("malformed object") // Options defines the configuration for this backend @@ -213,6 +269,272 @@ type Object struct { timestamp time.Time } +// Command the backend to run a named command +// +// The command run is name +// args may be used to read arguments from +// opts may be used to read optional arguments from +func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) { + switch name { + case "info": + return f.infoCmd(ctx, arg, opt) + default: + return nil, fs.ErrorCommandNotFound + } +} + +func (f *Fs) containerInfo(ctx context.Context, cnrID cid.ID) (container.Container, error) { + prm := pool.PrmContainerGet{ + ContainerID: cnrID, + } + var cnr container.Container + cnr, err := f.pool.GetContainer(ctx, prm) + if err != nil { + return container.Container{}, fmt.Errorf("couldn't get container '%s': %w", cnrID, err) + } + return cnr, err +} + +func (f *Fs) getObjectsHead(ctx context.Context, cnrID cid.ID, objIDs []oid.ID) ([]object.Object, error) { + var res []object.Object + for _, objID := range objIDs { + var prmHead pool.PrmObjectHead + prmHead.SetAddress(newAddress(cnrID, objID)) + + obj, err := f.pool.HeadObject(ctx, prmHead) + if err != nil { + return nil, err + } + res = append(res, obj) + } + + return res, nil +} + +type printer struct { + w io.Writer + err error +} + +func newPrinter(w io.Writer) *printer { + return &printer{w: w} +} + +func (p *printer) printf(format string, a ...interface{}) { + if p.err != nil { + return + } + if _, err := fmt.Fprintf(p.w, format, a...); err != nil { + p.err = err + } +} + +func (p *printer) lastError() error { + return p.err +} + +func (p *printer) printContainerInfo(cnrID cid.ID, cnr container.Container) { + p.printf("CID: %v\nOwner ID: %v", cnrID, cnr.Owner()) + var timestamp time.Time + var attrs []string + cnr.IterateAttributes(func(key string, value string) { + attrs = append(attrs, fmt.Sprintf(" %v=%v", key, value)) + if key == object.AttributeTimestamp { + val, err := strconv.ParseInt(value, 10, 64) + if err == nil { + timestamp = time.Unix(val, 0) + } + } + }) + if !timestamp.IsZero() { + p.printf("\nCreated: %v", timestamp) + } + if len(attrs) > 0 { + p.printf("\nAttributes:\n%s", strings.Join(attrs, "\n")) + } + + s := bytes.NewBufferString("") + if err := cnr.PlacementPolicy().WriteStringTo(s); err != nil { + return + } + + p.printf("\nPlacement policy:\n%s", s.String()) +} + +func (p *printer) printChecksum(name string, recv func() (checksum.Checksum, bool)) { + var strVal string + + cs, csSet := recv() + if csSet { + strVal = hex.EncodeToString(cs.Value()) + } else { + strVal = "" + } + + p.printf("\n%s: %s", name, strVal) +} + +func (p *printer) printObject(obj *object.Object) { + objIDStr := "" + cnrIDStr := objIDStr + if objID, ok := obj.ID(); ok { + objIDStr = objID.String() + } + if cnrID, ok := obj.ContainerID(); ok { + cnrIDStr = cnrID.String() + } + p.printf("\nID: %v", objIDStr) + p.printf("\nCID: %v", cnrIDStr) + p.printf("\nOwner: %s", obj.OwnerID()) + p.printf("\nCreatedAt: %d", obj.CreationEpoch()) + p.printf("\nSize: %d", obj.PayloadSize()) + p.printChecksum("HomoHash", obj.PayloadHomomorphicHash) + p.printChecksum("Checksum", obj.PayloadChecksum) + p.printf("\nType: %s", obj.Type()) + + p.printf("\nAttributes:") + for _, attr := range obj.Attributes() { + if attr.Key() == object.AttributeTimestamp { + var strVal string + val, err := strconv.ParseInt(attr.Value(), 10, 64) + if err == nil { + strVal = time.Unix(val, 0).String() + } else { + strVal = "malformed" + } + p.printf("\n %s=%s (%s)", + attr.Key(), + attr.Value(), + strVal) + continue + } + p.printf("\n %s=%s", attr.Key(), attr.Value()) + } + + if signature := obj.Signature(); signature != nil { + p.printf("\nID signature:") + + var sigV2 refs.Signature + signature.WriteToV2(&sigV2) + + p.printf("\n public key: %s", hex.EncodeToString(sigV2.GetKey())) + p.printf("\n signature: %s", hex.EncodeToString(sigV2.GetSign())) + } + + if ecHeader := obj.ECHeader(); ecHeader != nil { + p.printf("\nEC header:") + + p.printf("\n parent object ID: %s", ecHeader.Parent().EncodeToString()) + p.printf("\n index: %d", ecHeader.Index()) + p.printf("\n total: %d", ecHeader.Total()) + p.printf("\n header length: %d", ecHeader.HeaderLength()) + } + + p.printSplitHeader(obj) +} + +func (p *printer) printSplitHeader(obj *object.Object) { + if splitID := obj.SplitID(); splitID != nil { + p.printf("Split ID: %s\n", splitID) + } + + if objID, ok := obj.ParentID(); ok { + p.printf("Split ParentID: %s\n", objID) + } + + if prev, ok := obj.PreviousID(); ok { + p.printf("\nSplit PreviousID: %s", prev) + } + + for _, child := range obj.Children() { + p.printf("\nSplit ChildID: %s", child.String()) + } + + parent := obj.Parent() + if parent != nil { + p.printf("\n\nSplit Parent Header:") + + p.printObject(parent) + } + +} + +func formattedInfoOutput(format string, cnrID cid.ID, objHeads []object.Object) (string, error) { + format = strings.ReplaceAll(format, "{cid}", cnrID.String()) + objIDStr := "" + if len(objHeads) > 0 { + objID, ok := objHeads[0].ID() + if ok { + objIDStr = objID.String() + } + } + + return strings.ReplaceAll(format, "{oid}", objIDStr), nil +} + +func (f *Fs) infoCmd(ctx context.Context, arg []string, opt map[string]string) (out interface{}, err error) { + var cnrID cid.ID + + if cnrID, err = f.resolveContainerID(ctx, f.rootContainer); err != nil { + return nil, err + } + + var format string + for k, v := range opt { + switch k { + case "format": + format = v + default: + return nil, fmt.Errorf("unknown option \"%s\"", k) + } + } + + var objIDs []oid.ID + var filePath string + if len(arg) > 0 { + filePath = strings.TrimPrefix(arg[0], "/") + if f.rootDirectory != "" { + filePath = f.rootDirectory + "/" + filePath + } + + if objIDs, err = f.findObjectsFilePath(ctx, cnrID, filePath); err != nil { + return + } + } + + cnr, err := f.containerInfo(ctx, cnrID) + if err != nil { + return + } + var objHeads []object.Object + if objHeads, err = f.getObjectsHead(ctx, cnrID, objIDs); err != nil { + return + } + + if format != "" { + return formattedInfoOutput(format, cnrID, objHeads) + } + + w := bytes.NewBufferString("") + p := newPrinter(w) + p.printf(" --- Container info ---\n") + p.printContainerInfo(cnrID, cnr) + + if len(arg) > 0 { + p.printf("\n\n --- Object info ---") + if len(objHeads) > 0 { + // Print info about the first object only + p.printObject(&objHeads[0]) + } else { + p.printf("\nNo object with \"%s\" file path was found", filePath) + } + } + if err := p.lastError(); err != nil { + return nil, err + } + return w.String(), nil +} + // Shutdown the backend, closing any background tasks and any // cached connections. func (f *Fs) Shutdown(_ context.Context) error { -- 2.45.3 From 4733d46d839d2d48ead94de034dfa7a61f248222 Mon Sep 17 00:00:00 2001 From: Aleksey Kravchenko Date: Tue, 18 Feb 2025 09:58:29 +0300 Subject: [PATCH 2/2] [#13] Add docs about info FrostFS backend cmd Signed-off-by: Aleksey Kravchenko --- MANUAL.html | 211 ++++++++++++++++++++------------ MANUAL.md | 73 ++++++++++- MANUAL.txt | 76 +++++++++++- docs/content/commands/rclone.md | 2 +- docs/content/flags.md | 2 +- docs/content/frostfs.md | 69 +++++++++++ rclone.1 | 92 +++++++++++++- 7 files changed, 440 insertions(+), 85 deletions(-) diff --git a/MANUAL.html b/MANUAL.html index eb4138e2b..6954adaff 100644 --- a/MANUAL.html +++ b/MANUAL.html @@ -233,7 +233,7 @@

rclone(1) User Manual

Nick Craig-Wood

-

Jan 23, 2025

+

Feb 22, 2025

Rclone syncs your files to cloud storage

@@ -19294,7 +19294,7 @@ split into groups.

--tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --use-cookies Enable session cookiejar - --user-agent string Set the user-agent to a specified string (default "rclone/v1.68.2-beta.8335.a85292ca0.feature/add-container-zones-support") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.68.2-beta.8339.71f0cdbd4.feat/frostfs_info_cmd")

Performance

Flags helpful for increasing performance.

      --buffer-size SizeSuffix   In memory buffer size when reading files for each --transfer (default 16Mi)
@@ -33155,6 +33155,63 @@ decentralized object storage FrostFS).

  • Type: string
  • Required: false
  • +

    Backend commands

    +

    Here are the commands specific to the frostfs backend.

    +

    Run them with

    +
    rclone backend COMMAND remote:
    +

    The help below will explain what arguments each command takes.

    +

    See the backend command +for more info on how to pass options and arguments.

    +

    These can be run on a running backend using the rc command backend/command.

    +

    info

    +

    Show information about the FrostFS objects and containers

    +
    rclone backend info remote: [options] [<arguments>+]
    +

    This command can be used to get information about the FrostFS objects +and containers.

    +

    Usage Examples:

    +
    rclone backend info frostfs:container/path/to/dir
    +rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt
    +rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt -o "format={cid}:{oid}"
    +

    The optional "format" flag overrides the information output. In this +example, if an object is stored in a container with the identifier +"9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1" and its own identifier is +"4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP", the output of this +command will be +"9mvN7hsucoGoHjxPqrWmDipnMaemGVDqrxxPyynn1:4VpcNFsZqsQt1Gnfw2utBnzn5Blgc7i4kvtuXyKyJp".

    +

    The default output format is the same as that of the frostfs-cli +utility, with the "container get" and "object head" options. Here is an +example of output:

    +
     --- Container info ---
    +CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1
    +Owner ID: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX
    +Created: 2025-02-17 15:07:51 +0300 MSK
    +Attributes:
    +  Timestamp=1739794071
    +  Name=test
    +  __SYSTEM__NAME=test
    +  __SYSTEM__ZONE=container
    +  __SYSTEM__DISABLE_HOMOMORPHIC_HASHING=true
    +Placement policy:
    +REP 3
    +
    + --- Object info ---
    +ID: 4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP
    +CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1
    +Owner: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX
    +CreatedAt: 559
    +Size: 402905
    +HomoHash: <empty>
    +Checksum: 2a068fe24c53bc8bf7d6bbb997414f7938b080305dc45f9fd3ff684bc11fbb7b
    +Type: REGULAR
    +Attributes:
    +  FileName=cat.png
    +  FilePath=/dir1/dir2/dir3/cat.png
    +  Timestamp=1733410524 (2024-12-05 17:55:24 +0300 MSK)
    +ID signature:
    +  public key: 026b7c7a7a16225eb13a5a733495a1bcdd1f016dfa9193498821379b0de2ba6870
    +  signature: 049f6712c8378d323269b605a282bcacd7565ce2eefe1f10a9739c48945f739d95102c478b9cb1d429cd3330b4b5262e725392e322de3bbfa4ce18a9c842289219

    FTP

    FTP is the File Transfer Protocol. Rclone FTP support is provided using the

    See the metadata docs for more info.

    -

    Backend commands

    +

    Backend commands

    Here are the commands specific to the drive backend.

    Run them with

    rclone backend COMMAND remote:
    @@ -37209,7 +37266,7 @@ default).

    written.

    See the metadata docs for more info.

    -

    Backend commands

    +

    Backend commands

    Here are the commands specific to the hasher backend.

    Run them with

    rclone backend COMMAND remote:
    @@ -38111,7 +38168,7 @@ may be in the listing.

  • Type: string
  • Required: false
  • -

    Backend commands

    +

    Backend commands

    Here are the commands specific to the http backend.

    Run them with

    rclone backend COMMAND remote:
    @@ -40745,7 +40802,7 @@ provided primarily for debugging purposes.

  • Type: string
  • Required: false
  • -

    Backend commands

    +

    Backend commands

    Here are the commands specific to the netstorage backend.

    Run them with

    rclone backend COMMAND remote:
    @@ -43092,69 +43149,69 @@ href="https://learn.microsoft.com/en-us/onedrive/developer/rest-api/resources/pe API, which differs slightly between OneDrive Personal and Business.

    Example for OneDrive Personal:

    -
    [
    -    {
    -        "id": "1234567890ABC!123",
    -        "grantedTo": {
    -            "user": {
    -                "id": "ryan@contoso.com"
    -            },
    -            "application": {},
    -            "device": {}
    -        },
    -        "invitation": {
    -            "email": "ryan@contoso.com"
    -        },
    -        "link": {
    -            "webUrl": "https://1drv.ms/t/s!1234567890ABC"
    -        },
    -        "roles": [
    -            "read"
    -        ],
    -        "shareId": "s!1234567890ABC"
    -    }
    -]
    +
    [
    +    {
    +        "id": "1234567890ABC!123",
    +        "grantedTo": {
    +            "user": {
    +                "id": "ryan@contoso.com"
    +            },
    +            "application": {},
    +            "device": {}
    +        },
    +        "invitation": {
    +            "email": "ryan@contoso.com"
    +        },
    +        "link": {
    +            "webUrl": "https://1drv.ms/t/s!1234567890ABC"
    +        },
    +        "roles": [
    +            "read"
    +        ],
    +        "shareId": "s!1234567890ABC"
    +    }
    +]

    Example for OneDrive Business:

    -
    [
    -    {
    -        "id": "48d31887-5fad-4d73-a9f5-3c356e68a038",
    -        "grantedToIdentities": [
    -            {
    -                "user": {
    -                    "displayName": "ryan@contoso.com"
    -                },
    -                "application": {},
    -                "device": {}
    -            }
    -        ],
    -        "link": {
    -            "type": "view",
    -            "scope": "users",
    -            "webUrl": "https://contoso.sharepoint.com/:w:/t/design/a577ghg9hgh737613bmbjf839026561fmzhsr85ng9f3hjck2t5s"
    -        },
    -        "roles": [
    -            "read"
    -        ],
    -        "shareId": "u!LKj1lkdlals90j1nlkascl"
    -    },
    -    {
    -        "id": "5D33DD65C6932946",
    -        "grantedTo": {
    -            "user": {
    -                "displayName": "John Doe",
    -                "id": "efee1b77-fb3b-4f65-99d6-274c11914d12"
    -            },
    -            "application": {},
    -            "device": {}
    -        },
    -        "roles": [
    -            "owner"
    -        ],
    -        "shareId": "FWxc1lasfdbEAGM5fI7B67aB5ZMPDMmQ11U"
    -    }
    -]
    +
    [
    +    {
    +        "id": "48d31887-5fad-4d73-a9f5-3c356e68a038",
    +        "grantedToIdentities": [
    +            {
    +                "user": {
    +                    "displayName": "ryan@contoso.com"
    +                },
    +                "application": {},
    +                "device": {}
    +            }
    +        ],
    +        "link": {
    +            "type": "view",
    +            "scope": "users",
    +            "webUrl": "https://contoso.sharepoint.com/:w:/t/design/a577ghg9hgh737613bmbjf839026561fmzhsr85ng9f3hjck2t5s"
    +        },
    +        "roles": [
    +            "read"
    +        ],
    +        "shareId": "u!LKj1lkdlals90j1nlkascl"
    +    },
    +    {
    +        "id": "5D33DD65C6932946",
    +        "grantedTo": {
    +            "user": {
    +                "displayName": "John Doe",
    +                "id": "efee1b77-fb3b-4f65-99d6-274c11914d12"
    +            },
    +            "application": {},
    +            "device": {}
    +        },
    +        "roles": [
    +            "owner"
    +        ],
    +        "shareId": "FWxc1lasfdbEAGM5fI7B67aB5ZMPDMmQ11U"
    +    }
    +]

    To write permissions, pass in a "permissions" metadata key using this same format. The --metadata-mapper @@ -43168,12 +43225,12 @@ for a user. Creating a Public Link is also supported, if Link.Scope is set to "anonymous".

    Example request to add a "read" permission with --metadata-mapper:

    -
    {
    -    "Metadata": {
    -        "permissions": "[{\"grantedToIdentities\":[{\"user\":{\"id\":\"ryan@contoso.com\"}}],\"roles\":[\"read\"]}]"
    -    }
    -}
    +
    {
    +    "Metadata": {
    +        "permissions": "[{\"grantedToIdentities\":[{\"user\":{\"id\":\"ryan@contoso.com\"}}],\"roles\":[\"read\"]}]"
    +    }
    +}

    Note that adding a permission can fail if a conflicting permission already exists for the file/folder.

    To update an existing permission, include both the Permission ID and @@ -44502,7 +44559,7 @@ Encryption

  • Type: string
  • Required: false
  • -

    Backend commands

    +

    Backend commands

    Here are the commands specific to the oracleobjectstorage backend.

    Run them with

    @@ -46470,7 +46527,7 @@ Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,Righ
  • Type: string
  • Required: false
  • -

    Backend commands

    +

    Backend commands

    Here are the commands specific to the pikpak backend.

    Run them with

    rclone backend COMMAND remote:
    @@ -52315,7 +52372,7 @@ backend.

    See the metadata docs for more info.

    -

    Backend commands

    +

    Backend commands

    Here are the commands specific to the local backend.

    Run them with

    rclone backend COMMAND remote:
    diff --git a/MANUAL.md b/MANUAL.md index d3c9c94ab..b1ec1e28c 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -1,6 +1,6 @@ % rclone(1) User Manual % Nick Craig-Wood -% Jan 23, 2025 +% Feb 22, 2025 # Rclone syncs your files to cloud storage @@ -20498,7 +20498,7 @@ Flags for general networking and HTTP stuff. --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --use-cookies Enable session cookiejar - --user-agent string Set the user-agent to a specified string (default "rclone/v1.68.2-beta.8335.a85292ca0.feature/add-container-zones-support") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.68.2-beta.8339.71f0cdbd4.feat/frostfs_info_cmd") ``` @@ -35104,6 +35104,75 @@ Properties: - Type: string - Required: false +## Backend commands + +Here are the commands specific to the frostfs backend. + +Run them with + + rclone backend COMMAND remote: + +The help below will explain what arguments each command takes. + +See the [backend](https://rclone.org/commands/rclone_backend/) command for more +info on how to pass options and arguments. + +These can be run on a running backend using the rc command +[backend/command](https://rclone.org/rc/#backend-command). + +### info + +Show information about the FrostFS objects and containers + + rclone backend info remote: [options] [+] + +This command can be used to get information about the FrostFS objects and containers. + +Usage Examples: + + rclone backend info frostfs:container/path/to/dir + rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt + rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt -o "format={cid}:{oid}" + +The optional "format" flag overrides the information output. In this example, if an object is stored in +a container with the identifier "9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1" and +its own identifier is "4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP", the output of this command will be +"9mvN7hsucoGoHjxPqrWmDipnMaemGVDqrxxPyynn1:4VpcNFsZqsQt1Gnfw2utBnzn5Blgc7i4kvtuXyKyJp". + +The default output format is the same as that of the frostfs-cli utility, +with the "container get" and "object head" options. Here is an example of output: + + --- Container info --- + CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 + Owner ID: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX + Created: 2025-02-17 15:07:51 +0300 MSK + Attributes: + Timestamp=1739794071 + Name=test + __SYSTEM__NAME=test + __SYSTEM__ZONE=container + __SYSTEM__DISABLE_HOMOMORPHIC_HASHING=true + Placement policy: + REP 3 + + --- Object info --- + ID: 4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP + CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 + Owner: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX + CreatedAt: 559 + Size: 402905 + HomoHash: + Checksum: 2a068fe24c53bc8bf7d6bbb997414f7938b080305dc45f9fd3ff684bc11fbb7b + Type: REGULAR + Attributes: + FileName=cat.png + FilePath=/dir1/dir2/dir3/cat.png + Timestamp=1733410524 (2024-12-05 17:55:24 +0300 MSK) + ID signature: + public key: 026b7c7a7a16225eb13a5a733495a1bcdd1f016dfa9193498821379b0de2ba6870 + signature: 049f6712c8378d323269b605a282bcacd7565ce2eefe1f10a9739c48945f739d95102c478b9cb1d429cd3330b4b5262e725392e322de3bbfa4ce18a9c842289219 + + # FTP diff --git a/MANUAL.txt b/MANUAL.txt index 1ea427688..2ac24eaac 100644 --- a/MANUAL.txt +++ b/MANUAL.txt @@ -1,6 +1,6 @@ rclone(1) User Manual Nick Craig-Wood -Jan 23, 2025 +Feb 22, 2025 Rclone syncs your files to cloud storage @@ -20166,7 +20166,7 @@ Flags for general networking and HTTP stuff. --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --use-cookies Enable session cookiejar - --user-agent string Set the user-agent to a specified string (default "rclone/v1.68.2-beta.8335.a85292ca0.feature/add-container-zones-support") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.68.2-beta.8339.71f0cdbd4.feat/frostfs_info_cmd") Performance @@ -34575,6 +34575,78 @@ Properties: - Type: string - Required: false +Backend commands + +Here are the commands specific to the frostfs backend. + +Run them with + + rclone backend COMMAND remote: + +The help below will explain what arguments each command takes. + +See the backend command for more info on how to pass options and +arguments. + +These can be run on a running backend using the rc command +backend/command. + +info + +Show information about the FrostFS objects and containers + + rclone backend info remote: [options] [+] + +This command can be used to get information about the FrostFS objects +and containers. + +Usage Examples: + + rclone backend info frostfs:container/path/to/dir + rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt + rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt -o "format={cid}:{oid}" + +The optional "format" flag overrides the information output. In this +example, if an object is stored in a container with the identifier +"9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1" and its own identifier is +"4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP", the output of this +command will be +"9mvN7hsucoGoHjxPqrWmDipnMaemGVDqrxxPyynn1:4VpcNFsZqsQt1Gnfw2utBnzn5Blgc7i4kvtuXyKyJp". + +The default output format is the same as that of the frostfs-cli +utility, with the "container get" and "object head" options. Here is an +example of output: + + --- Container info --- + CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 + Owner ID: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX + Created: 2025-02-17 15:07:51 +0300 MSK + Attributes: + Timestamp=1739794071 + Name=test + __SYSTEM__NAME=test + __SYSTEM__ZONE=container + __SYSTEM__DISABLE_HOMOMORPHIC_HASHING=true + Placement policy: + REP 3 + + --- Object info --- + ID: 4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP + CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 + Owner: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX + CreatedAt: 559 + Size: 402905 + HomoHash: + Checksum: 2a068fe24c53bc8bf7d6bbb997414f7938b080305dc45f9fd3ff684bc11fbb7b + Type: REGULAR + Attributes: + FileName=cat.png + FilePath=/dir1/dir2/dir3/cat.png + Timestamp=1733410524 (2024-12-05 17:55:24 +0300 MSK) + ID signature: + public key: 026b7c7a7a16225eb13a5a733495a1bcdd1f016dfa9193498821379b0de2ba6870 + signature: 049f6712c8378d323269b605a282bcacd7565ce2eefe1f10a9739c48945f739d95102c478b9cb1d429cd3330b4b5262e725392e322de3bbfa4ce18a9c842289219 + FTP FTP is the File Transfer Protocol. Rclone FTP support is provided using diff --git a/docs/content/commands/rclone.md b/docs/content/commands/rclone.md index d43e14ba8..d80a35a1c 100644 --- a/docs/content/commands/rclone.md +++ b/docs/content/commands/rclone.md @@ -945,7 +945,7 @@ rclone [flags] --use-json-log Use json log format --use-mmap Use mmap allocator (see docs) --use-server-modtime Use server modified time instead of object metadata - --user-agent string Set the user-agent to a specified string (default "rclone/v1.68.2-beta.8335.a85292ca0.feature/add-container-zones-support") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.68.2-beta.8339.71f0cdbd4.feat/frostfs_info_cmd") -v, --verbose count Print lots more stuff (repeat for more) -V, --version Print the version number --webdav-bearer-token string Bearer token instead of user/pass (e.g. a Macaroon) diff --git a/docs/content/flags.md b/docs/content/flags.md index af8048265..57fc15854 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -115,7 +115,7 @@ Flags for general networking and HTTP stuff. --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --use-cookies Enable session cookiejar - --user-agent string Set the user-agent to a specified string (default "rclone/v1.68.2-beta.8335.a85292ca0.feature/add-container-zones-support") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.68.2-beta.8339.71f0cdbd4.feat/frostfs_info_cmd") ``` diff --git a/docs/content/frostfs.md b/docs/content/frostfs.md index 13f6b0d27..8c5c9e294 100644 --- a/docs/content/frostfs.md +++ b/docs/content/frostfs.md @@ -487,4 +487,73 @@ Properties: - Type: string - Required: false +## Backend commands + +Here are the commands specific to the frostfs backend. + +Run them with + + rclone backend COMMAND remote: + +The help below will explain what arguments each command takes. + +See the [backend](/commands/rclone_backend/) command for more +info on how to pass options and arguments. + +These can be run on a running backend using the rc command +[backend/command](/rc/#backend-command). + +### info + +Show information about the FrostFS objects and containers + + rclone backend info remote: [options] [+] + +This command can be used to get information about the FrostFS objects and containers. + +Usage Examples: + + rclone backend info frostfs:container/path/to/dir + rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt + rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt -o "format={cid}:{oid}" + +The optional "format" flag overrides the information output. In this example, if an object is stored in +a container with the identifier "9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1" and +its own identifier is "4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP", the output of this command will be +"9mvN7hsucoGoHjxPqrWmDipnMaemGVDqrxxPyynn1:4VpcNFsZqsQt1Gnfw2utBnzn5Blgc7i4kvtuXyKyJp". + +The default output format is the same as that of the frostfs-cli utility, +with the "container get" and "object head" options. Here is an example of output: + + --- Container info --- + CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 + Owner ID: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX + Created: 2025-02-17 15:07:51 +0300 MSK + Attributes: + Timestamp=1739794071 + Name=test + __SYSTEM__NAME=test + __SYSTEM__ZONE=container + __SYSTEM__DISABLE_HOMOMORPHIC_HASHING=true + Placement policy: + REP 3 + + --- Object info --- + ID: 4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP + CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 + Owner: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX + CreatedAt: 559 + Size: 402905 + HomoHash: + Checksum: 2a068fe24c53bc8bf7d6bbb997414f7938b080305dc45f9fd3ff684bc11fbb7b + Type: REGULAR + Attributes: + FileName=cat.png + FilePath=/dir1/dir2/dir3/cat.png + Timestamp=1733410524 (2024-12-05 17:55:24 +0300 MSK) + ID signature: + public key: 026b7c7a7a16225eb13a5a733495a1bcdd1f016dfa9193498821379b0de2ba6870 + signature: 049f6712c8378d323269b605a282bcacd7565ce2eefe1f10a9739c48945f739d95102c478b9cb1d429cd3330b4b5262e725392e322de3bbfa4ce18a9c842289219 + + {{< rem autogenerated options stop >}} diff --git a/rclone.1 b/rclone.1 index a4c6ddc25..20a872a85 100644 --- a/rclone.1 +++ b/rclone.1 @@ -15,7 +15,7 @@ . ftr VB CB . ftr VBI CBI .\} -.TH "rclone" "1" "Jan 23, 2025" "User Manual" "" +.TH "rclone" "1" "Feb 22, 2025" "User Manual" "" .hy .SH Rclone syncs your files to cloud storage .PP @@ -27735,7 +27735,7 @@ Flags for general networking and HTTP stuff. --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --use-cookies Enable session cookiejar - --user-agent string Set the user-agent to a specified string (default \[dq]rclone/v1.68.2-beta.8335.a85292ca0.feature/add-container-zones-support\[dq]) + --user-agent string Set the user-agent to a specified string (default \[dq]rclone/v1.68.2-beta.8339.71f0cdbd4.feat/frostfs_info_cmd\[dq]) \f[R] .fi .SS Performance @@ -46090,6 +46090,94 @@ Env Var: RCLONE_FROSTFS_DESCRIPTION Type: string .IP \[bu] 2 Required: false +.SS Backend commands +.PP +Here are the commands specific to the frostfs backend. +.PP +Run them with +.IP +.nf +\f[C] +rclone backend COMMAND remote: +\f[R] +.fi +.PP +The help below will explain what arguments each command takes. +.PP +See the backend (https://rclone.org/commands/rclone_backend/) command +for more info on how to pass options and arguments. +.PP +These can be run on a running backend using the rc command +backend/command (https://rclone.org/rc/#backend-command). +.SS info +.PP +Show information about the FrostFS objects and containers +.IP +.nf +\f[C] +rclone backend info remote: [options] [+] +\f[R] +.fi +.PP +This command can be used to get information about the FrostFS objects +and containers. +.PP +Usage Examples: +.IP +.nf +\f[C] +rclone backend info frostfs:container/path/to/dir +rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt +rclone backend info frostfs:container/path/to/dir path/to/file/in/dir.txt -o \[dq]format={cid}:{oid}\[dq] +\f[R] +.fi +.PP +The optional \[dq]format\[dq] flag overrides the information output. +In this example, if an object is stored in a container with the +identifier \[dq]9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1\[dq] and +its own identifier is +\[dq]4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP\[dq], the output of +this command will be +\[dq]9mvN7hsucoGoHjxPqrWmDipnMaemGVDqrxxPyynn1:4VpcNFsZqsQt1Gnfw2utBnzn5Blgc7i4kvtuXyKyJp\[dq]. +.PP +The default output format is the same as that of the frostfs-cli +utility, with the \[dq]container get\[dq] and \[dq]object head\[dq] +options. +Here is an example of output: +.IP +.nf +\f[C] + --- Container info --- +CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 +Owner ID: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX +Created: 2025-02-17 15:07:51 +0300 MSK +Attributes: + Timestamp=1739794071 + Name=test + __SYSTEM__NAME=test + __SYSTEM__ZONE=container + __SYSTEM__DISABLE_HOMOMORPHIC_HASHING=true +Placement policy: +REP 3 + + --- Object info --- +ID: 4VPCNFsZ2SQt1GNfYw2uTBNnz5bLgC7i4k4ovtuXKyJP +CID: 9mvN7hsUcYoGoHjxpRWtqmDipnmaeRmGVDqRxxPyy2n1 +Owner: NQL7q6PvPaisWNwdWfoR1LsEsAyje8P3jX +CreatedAt: 559 +Size: 402905 +HomoHash: +Checksum: 2a068fe24c53bc8bf7d6bbb997414f7938b080305dc45f9fd3ff684bc11fbb7b +Type: REGULAR +Attributes: + FileName=cat.png + FilePath=/dir1/dir2/dir3/cat.png + Timestamp=1733410524 (2024-12-05 17:55:24 +0300 MSK) +ID signature: + public key: 026b7c7a7a16225eb13a5a733495a1bcdd1f016dfa9193498821379b0de2ba6870 + signature: 049f6712c8378d323269b605a282bcacd7565ce2eefe1f10a9739c48945f739d95102c478b9cb1d429cd3330b4b5262e725392e322de3bbfa4ce18a9c842289219 +\f[R] +.fi .SH FTP .PP FTP is the File Transfer Protocol. -- 2.45.3