lsjson: add --stat flag and operations/stat api
This enables information about single files to be efficiently retrieved.
This commit is contained in:
parent
3fbaa4c0b0
commit
f529c02446
5 changed files with 721 additions and 105 deletions
|
@ -9,13 +9,15 @@ import (
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/rclone/rclone/cmd"
|
"github.com/rclone/rclone/cmd"
|
||||||
"github.com/rclone/rclone/cmd/ls/lshelp"
|
"github.com/rclone/rclone/cmd/ls/lshelp"
|
||||||
|
"github.com/rclone/rclone/fs"
|
||||||
"github.com/rclone/rclone/fs/config/flags"
|
"github.com/rclone/rclone/fs/config/flags"
|
||||||
"github.com/rclone/rclone/fs/operations"
|
"github.com/rclone/rclone/fs/operations"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
opt operations.ListJSONOpt
|
opt operations.ListJSONOpt
|
||||||
|
statOnly bool
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
@ -30,6 +32,7 @@ func init() {
|
||||||
flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", false, "Show only files in the listing.")
|
flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", false, "Show only files in the listing.")
|
||||||
flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Show only directories in the listing.")
|
flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Show only directories in the listing.")
|
||||||
flags.StringArrayVarP(cmdFlags, &opt.HashTypes, "hash-type", "", nil, "Show only this hash type (may be repeated).")
|
flags.StringArrayVarP(cmdFlags, &opt.HashTypes, "hash-type", "", nil, "Show only this hash type (may be repeated).")
|
||||||
|
flags.BoolVarP(cmdFlags, &statOnly, "stat", "", false, "Just return the info for the pointed to file.")
|
||||||
}
|
}
|
||||||
|
|
||||||
var commandDefinition = &cobra.Command{
|
var commandDefinition = &cobra.Command{
|
||||||
|
@ -79,6 +82,12 @@ returned
|
||||||
If --files-only is not specified directories in addition to the files
|
If --files-only is not specified directories in addition to the files
|
||||||
will be returned.
|
will be returned.
|
||||||
|
|
||||||
|
if --stat is set then a single JSON blob will be returned about the
|
||||||
|
item pointed to. This will return an error if the item isn't found.
|
||||||
|
However on bucket based backends (like s3, gcs, b2, azureblob etc) if
|
||||||
|
the item isn't found it will return an empty directory as it isn't
|
||||||
|
possible to tell empty directories from missing directories there.
|
||||||
|
|
||||||
The Path field will only show folders below the remote path being listed.
|
The Path field will only show folders below the remote path being listed.
|
||||||
If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt"
|
If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt"
|
||||||
will be "subfolder/file.txt", not "remote:path/subfolder/file.txt".
|
will be "subfolder/file.txt", not "remote:path/subfolder/file.txt".
|
||||||
|
@ -99,33 +108,59 @@ will be shown ("2017-05-31T16:15:57+01:00").
|
||||||
The whole output can be processed as a JSON blob, or alternatively it
|
The whole output can be processed as a JSON blob, or alternatively it
|
||||||
can be processed line by line as each item is written one to a line.
|
can be processed line by line as each item is written one to a line.
|
||||||
` + lshelp.Help,
|
` + lshelp.Help,
|
||||||
Run: func(command *cobra.Command, args []string) {
|
RunE: func(command *cobra.Command, args []string) error {
|
||||||
cmd.CheckArgs(1, 1, command, args)
|
cmd.CheckArgs(1, 1, command, args)
|
||||||
fsrc := cmd.NewFsSrc(args)
|
var fsrc fs.Fs
|
||||||
|
var remote string
|
||||||
|
if statOnly {
|
||||||
|
fsrc, remote = cmd.NewFsFile(args[0])
|
||||||
|
} else {
|
||||||
|
fsrc = cmd.NewFsSrc(args)
|
||||||
|
}
|
||||||
cmd.Run(false, false, command, func() error {
|
cmd.Run(false, false, command, func() error {
|
||||||
fmt.Println("[")
|
if statOnly {
|
||||||
first := true
|
item, err := operations.StatJSON(context.Background(), fsrc, remote, &opt)
|
||||||
err := operations.ListJSON(context.Background(), fsrc, "", &opt, func(item *operations.ListJSONItem) error {
|
if err != nil {
|
||||||
out, err := json.Marshal(item)
|
return err
|
||||||
|
}
|
||||||
|
out, err := json.MarshalIndent(item, "", "\t")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "failed to marshal list object")
|
return errors.Wrap(err, "failed to marshal list object")
|
||||||
}
|
}
|
||||||
if first {
|
|
||||||
first = false
|
|
||||||
} else {
|
|
||||||
fmt.Print(",\n")
|
|
||||||
}
|
|
||||||
_, err = os.Stdout.Write(out)
|
_, err = os.Stdout.Write(out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "failed to write to output")
|
return errors.Wrap(err, "failed to write to output")
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if !first {
|
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
} else {
|
||||||
|
fmt.Println("[")
|
||||||
|
first := true
|
||||||
|
err := operations.ListJSON(context.Background(), fsrc, remote, &opt, func(item *operations.ListJSONItem) error {
|
||||||
|
out, err := json.Marshal(item)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "failed to marshal list object")
|
||||||
|
}
|
||||||
|
if first {
|
||||||
|
first = false
|
||||||
|
} else {
|
||||||
|
fmt.Print(",\n")
|
||||||
|
}
|
||||||
|
_, err = os.Stdout.Write(out)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "failed to write to output")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !first {
|
||||||
|
fmt.Println()
|
||||||
|
}
|
||||||
|
fmt.Println("]")
|
||||||
}
|
}
|
||||||
fmt.Println("]")
|
return nil
|
||||||
return err
|
|
||||||
})
|
})
|
||||||
|
return nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ package operations
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"path"
|
"path"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
@ -81,115 +82,169 @@ type ListJSONOpt struct {
|
||||||
HashTypes []string `json:"hashTypes"` // hash types to show if ShowHash is set, e.g. "MD5", "SHA-1"
|
HashTypes []string `json:"hashTypes"` // hash types to show if ShowHash is set, e.g. "MD5", "SHA-1"
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListJSON lists fsrc using the options in opt calling callback for each item
|
// state for ListJson
|
||||||
func ListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJSONItem) error) error {
|
type listJSON struct {
|
||||||
var cipher *crypt.Cipher
|
fsrc fs.Fs
|
||||||
|
remote string
|
||||||
|
format string
|
||||||
|
opt *ListJSONOpt
|
||||||
|
cipher *crypt.Cipher
|
||||||
|
hashTypes []hash.Type
|
||||||
|
dirs bool
|
||||||
|
files bool
|
||||||
|
canGetTier bool
|
||||||
|
isBucket bool
|
||||||
|
showHash bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt) (*listJSON, error) {
|
||||||
|
lj := &listJSON{
|
||||||
|
fsrc: fsrc,
|
||||||
|
remote: remote,
|
||||||
|
opt: opt,
|
||||||
|
dirs: true,
|
||||||
|
files: true,
|
||||||
|
}
|
||||||
|
// Dirs Files
|
||||||
|
// !FilesOnly,!DirsOnly true true
|
||||||
|
// !FilesOnly,DirsOnly true false
|
||||||
|
// FilesOnly,!DirsOnly false true
|
||||||
|
// FilesOnly,DirsOnly true true
|
||||||
|
if !opt.FilesOnly && opt.DirsOnly {
|
||||||
|
lj.files = false
|
||||||
|
} else if opt.FilesOnly && !opt.DirsOnly {
|
||||||
|
lj.dirs = false
|
||||||
|
}
|
||||||
if opt.ShowEncrypted {
|
if opt.ShowEncrypted {
|
||||||
fsInfo, _, _, config, err := fs.ConfigFs(fsrc.Name() + ":" + fsrc.Root())
|
fsInfo, _, _, config, err := fs.ConfigFs(fsrc.Name() + ":" + fsrc.Root())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "ListJSON failed to load config for crypt remote")
|
return nil, errors.Wrap(err, "ListJSON failed to load config for crypt remote")
|
||||||
}
|
}
|
||||||
if fsInfo.Name != "crypt" {
|
if fsInfo.Name != "crypt" {
|
||||||
return errors.New("The remote needs to be of type \"crypt\"")
|
return nil, errors.New("The remote needs to be of type \"crypt\"")
|
||||||
}
|
}
|
||||||
cipher, err = crypt.NewCipher(config)
|
lj.cipher, err = crypt.NewCipher(config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "ListJSON failed to make new crypt remote")
|
return nil, errors.Wrap(err, "ListJSON failed to make new crypt remote")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
features := fsrc.Features()
|
features := fsrc.Features()
|
||||||
canGetTier := features.GetTier
|
lj.canGetTier = features.GetTier
|
||||||
format := formatForPrecision(fsrc.Precision())
|
lj.format = formatForPrecision(fsrc.Precision())
|
||||||
isBucket := features.BucketBased && remote == "" && fsrc.Root() == "" // if bucket based remote listing the root mark directories as buckets
|
lj.isBucket = features.BucketBased && remote == "" && fsrc.Root() == "" // if bucket based remote listing the root mark directories as buckets
|
||||||
showHash := opt.ShowHash
|
lj.showHash = opt.ShowHash
|
||||||
hashTypes := fsrc.Hashes().Array()
|
lj.hashTypes = fsrc.Hashes().Array()
|
||||||
if len(opt.HashTypes) != 0 {
|
if len(opt.HashTypes) != 0 {
|
||||||
showHash = true
|
lj.showHash = true
|
||||||
hashTypes = []hash.Type{}
|
lj.hashTypes = []hash.Type{}
|
||||||
for _, hashType := range opt.HashTypes {
|
for _, hashType := range opt.HashTypes {
|
||||||
var ht hash.Type
|
var ht hash.Type
|
||||||
err := ht.Set(hashType)
|
err := ht.Set(hashType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
hashTypes = append(hashTypes, ht)
|
lj.hashTypes = append(lj.hashTypes, ht)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
err := walk.ListR(ctx, fsrc, remote, false, ConfigMaxDepth(ctx, opt.Recurse), walk.ListAll, func(entries fs.DirEntries) (err error) {
|
return lj, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert a single entry to JSON
|
||||||
|
//
|
||||||
|
// It may return nil if there is no entry to return
|
||||||
|
func (lj *listJSON) entry(ctx context.Context, entry fs.DirEntry) (*ListJSONItem, error) {
|
||||||
|
switch entry.(type) {
|
||||||
|
case fs.Directory:
|
||||||
|
if lj.opt.FilesOnly {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
case fs.Object:
|
||||||
|
if lj.opt.DirsOnly {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
fs.Errorf(nil, "Unknown type %T in listing", entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
item := &ListJSONItem{
|
||||||
|
Path: entry.Remote(),
|
||||||
|
Name: path.Base(entry.Remote()),
|
||||||
|
Size: entry.Size(),
|
||||||
|
}
|
||||||
|
if entry.Remote() == "" {
|
||||||
|
item.Name = ""
|
||||||
|
}
|
||||||
|
if !lj.opt.NoModTime {
|
||||||
|
item.ModTime = Timestamp{When: entry.ModTime(ctx), Format: lj.format}
|
||||||
|
}
|
||||||
|
if !lj.opt.NoMimeType {
|
||||||
|
item.MimeType = fs.MimeTypeDirEntry(ctx, entry)
|
||||||
|
}
|
||||||
|
if lj.cipher != nil {
|
||||||
|
switch entry.(type) {
|
||||||
|
case fs.Directory:
|
||||||
|
item.EncryptedPath = lj.cipher.EncryptDirName(entry.Remote())
|
||||||
|
case fs.Object:
|
||||||
|
item.EncryptedPath = lj.cipher.EncryptFileName(entry.Remote())
|
||||||
|
default:
|
||||||
|
fs.Errorf(nil, "Unknown type %T in listing", entry)
|
||||||
|
}
|
||||||
|
item.Encrypted = path.Base(item.EncryptedPath)
|
||||||
|
}
|
||||||
|
if do, ok := entry.(fs.IDer); ok {
|
||||||
|
item.ID = do.ID()
|
||||||
|
}
|
||||||
|
if o, ok := entry.(fs.Object); lj.opt.ShowOrigIDs && ok {
|
||||||
|
if do, ok := fs.UnWrapObject(o).(fs.IDer); ok {
|
||||||
|
item.OrigID = do.ID()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch x := entry.(type) {
|
||||||
|
case fs.Directory:
|
||||||
|
item.IsDir = true
|
||||||
|
item.IsBucket = lj.isBucket
|
||||||
|
case fs.Object:
|
||||||
|
item.IsDir = false
|
||||||
|
if lj.showHash {
|
||||||
|
item.Hashes = make(map[string]string)
|
||||||
|
for _, hashType := range lj.hashTypes {
|
||||||
|
hash, err := x.Hash(ctx, hashType)
|
||||||
|
if err != nil {
|
||||||
|
fs.Errorf(x, "Failed to read hash: %v", err)
|
||||||
|
} else if hash != "" {
|
||||||
|
item.Hashes[hashType.String()] = hash
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if lj.canGetTier {
|
||||||
|
if do, ok := x.(fs.GetTierer); ok {
|
||||||
|
item.Tier = do.GetTier()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
fs.Errorf(nil, "Unknown type %T in listing in ListJSON", entry)
|
||||||
|
}
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListJSON lists fsrc using the options in opt calling callback for each item
|
||||||
|
func ListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJSONItem) error) error {
|
||||||
|
lj, err := newListJSON(ctx, fsrc, remote, opt)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = walk.ListR(ctx, fsrc, remote, false, ConfigMaxDepth(ctx, lj.opt.Recurse), walk.ListAll, func(entries fs.DirEntries) (err error) {
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
switch entry.(type) {
|
item, err := lj.entry(ctx, entry)
|
||||||
case fs.Directory:
|
|
||||||
if opt.FilesOnly {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
case fs.Object:
|
|
||||||
if opt.DirsOnly {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
fs.Errorf(nil, "Unknown type %T in listing", entry)
|
|
||||||
}
|
|
||||||
|
|
||||||
item := ListJSONItem{
|
|
||||||
Path: entry.Remote(),
|
|
||||||
Name: path.Base(entry.Remote()),
|
|
||||||
Size: entry.Size(),
|
|
||||||
}
|
|
||||||
if !opt.NoModTime {
|
|
||||||
item.ModTime = Timestamp{When: entry.ModTime(ctx), Format: format}
|
|
||||||
}
|
|
||||||
if !opt.NoMimeType {
|
|
||||||
item.MimeType = fs.MimeTypeDirEntry(ctx, entry)
|
|
||||||
}
|
|
||||||
if cipher != nil {
|
|
||||||
switch entry.(type) {
|
|
||||||
case fs.Directory:
|
|
||||||
item.EncryptedPath = cipher.EncryptDirName(entry.Remote())
|
|
||||||
case fs.Object:
|
|
||||||
item.EncryptedPath = cipher.EncryptFileName(entry.Remote())
|
|
||||||
default:
|
|
||||||
fs.Errorf(nil, "Unknown type %T in listing", entry)
|
|
||||||
}
|
|
||||||
item.Encrypted = path.Base(item.EncryptedPath)
|
|
||||||
}
|
|
||||||
if do, ok := entry.(fs.IDer); ok {
|
|
||||||
item.ID = do.ID()
|
|
||||||
}
|
|
||||||
if o, ok := entry.(fs.Object); opt.ShowOrigIDs && ok {
|
|
||||||
if do, ok := fs.UnWrapObject(o).(fs.IDer); ok {
|
|
||||||
item.OrigID = do.ID()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
switch x := entry.(type) {
|
|
||||||
case fs.Directory:
|
|
||||||
item.IsDir = true
|
|
||||||
item.IsBucket = isBucket
|
|
||||||
case fs.Object:
|
|
||||||
item.IsDir = false
|
|
||||||
if showHash {
|
|
||||||
item.Hashes = make(map[string]string)
|
|
||||||
for _, hashType := range hashTypes {
|
|
||||||
hash, err := x.Hash(ctx, hashType)
|
|
||||||
if err != nil {
|
|
||||||
fs.Errorf(x, "Failed to read hash: %v", err)
|
|
||||||
} else if hash != "" {
|
|
||||||
item.Hashes[hashType.String()] = hash
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if canGetTier {
|
|
||||||
if do, ok := x.(fs.GetTierer); ok {
|
|
||||||
item.Tier = do.GetTier()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
fs.Errorf(nil, "Unknown type %T in listing in ListJSON", entry)
|
|
||||||
}
|
|
||||||
err = callback(&item)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "callback failed in ListJSON")
|
return errors.Wrap(err, "creating entry failed in ListJSON")
|
||||||
|
}
|
||||||
|
if item != nil {
|
||||||
|
err = callback(item)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "callback failed in ListJSON")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
@ -198,3 +253,72 @@ func ListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt,
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StatJSON returns a single JSON stat entry for the fsrc, remote path
|
||||||
|
//
|
||||||
|
// The item returned may be nil if it is not found or excluded with DirsOnly/FilesOnly
|
||||||
|
func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt) (item *ListJSONItem, err error) {
|
||||||
|
// FIXME this could me more efficient we had a new primitive
|
||||||
|
// NewDirEntry() which returned an Object or a Directory
|
||||||
|
lj, err := newListJSON(ctx, fsrc, remote, opt)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Could be a file or a directory here
|
||||||
|
if lj.files {
|
||||||
|
// NewObject can return the sentinel errors ErrorObjectNotFound or ErrorIsDir
|
||||||
|
// ErrorObjectNotFound can mean the source is a directory or not found
|
||||||
|
obj, err := fsrc.NewObject(ctx, remote)
|
||||||
|
if err == fs.ErrorObjectNotFound {
|
||||||
|
if !lj.dirs {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
} else if err == fs.ErrorIsDir {
|
||||||
|
if !lj.dirs {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
// This could return a made up ListJSONItem here
|
||||||
|
// but that wouldn't have the IDs etc in
|
||||||
|
} else if err != nil {
|
||||||
|
if !lj.dirs {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return lj.entry(ctx, obj)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Must be a directory here
|
||||||
|
if remote == "" {
|
||||||
|
// Check the root directory exists
|
||||||
|
_, err := fsrc.List(ctx, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return lj.entry(ctx, fs.NewDir("", time.Now()))
|
||||||
|
}
|
||||||
|
parent := path.Dir(remote)
|
||||||
|
if parent == "." || parent == "/" {
|
||||||
|
parent = ""
|
||||||
|
}
|
||||||
|
entries, err := fsrc.List(ctx, parent)
|
||||||
|
if err == fs.ErrorDirNotFound {
|
||||||
|
return nil, nil
|
||||||
|
} else if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
equal := func(a, b string) bool { return a == b }
|
||||||
|
if fsrc.Features().CaseInsensitive {
|
||||||
|
equal = strings.EqualFold
|
||||||
|
}
|
||||||
|
var foundEntry fs.DirEntry
|
||||||
|
for _, entry := range entries {
|
||||||
|
if equal(entry.Remote(), remote) {
|
||||||
|
foundEntry = entry
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if foundEntry == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return lj.entry(ctx, foundEntry)
|
||||||
|
}
|
||||||
|
|
355
fs/operations/lsjson_test.go
Normal file
355
fs/operations/lsjson_test.go
Normal file
|
@ -0,0 +1,355 @@
|
||||||
|
package operations_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sort"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rclone/rclone/fs"
|
||||||
|
"github.com/rclone/rclone/fs/operations"
|
||||||
|
"github.com/rclone/rclone/fstest"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Compare a and b in a file system idependent way
|
||||||
|
func compareListJSONItem(t *testing.T, a, b *operations.ListJSONItem, precision time.Duration) {
|
||||||
|
assert.Equal(t, a.Path, b.Path, "Path")
|
||||||
|
assert.Equal(t, a.Name, b.Name, "Name")
|
||||||
|
// assert.Equal(t, a.EncryptedPath, b.EncryptedPath, "EncryptedPath")
|
||||||
|
// assert.Equal(t, a.Encrypted, b.Encrypted, "Encrypted")
|
||||||
|
if !a.IsDir {
|
||||||
|
assert.Equal(t, a.Size, b.Size, "Size")
|
||||||
|
}
|
||||||
|
// assert.Equal(t, a.MimeType, a.Mib.MimeType, "MimeType")
|
||||||
|
if !a.IsDir {
|
||||||
|
fstest.AssertTimeEqualWithPrecision(t, "ListJSON", a.ModTime.When, b.ModTime.When, precision)
|
||||||
|
}
|
||||||
|
assert.Equal(t, a.IsDir, b.IsDir, "IsDir")
|
||||||
|
// assert.Equal(t, a.Hashes, a.b.Hashes, "Hashes")
|
||||||
|
// assert.Equal(t, a.ID, b.ID, "ID")
|
||||||
|
// assert.Equal(t, a.OrigID, a.b.OrigID, "OrigID")
|
||||||
|
// assert.Equal(t, a.Tier, b.Tier, "Tier")
|
||||||
|
// assert.Equal(t, a.IsBucket, a.Isb.IsBucket, "IsBucket")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListJSON(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
r := fstest.NewRun(t)
|
||||||
|
defer r.Finalise()
|
||||||
|
file1 := r.WriteBoth(ctx, "file1", "file1", t1)
|
||||||
|
file2 := r.WriteBoth(ctx, "sub/file2", "sub/file2", t2)
|
||||||
|
|
||||||
|
fstest.CheckItems(t, r.Fremote, file1, file2)
|
||||||
|
precision := fs.GetModifyWindow(ctx, r.Fremote)
|
||||||
|
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
remote string
|
||||||
|
opt operations.ListJSONOpt
|
||||||
|
want []*operations.ListJSONItem
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Default",
|
||||||
|
opt: operations.ListJSONOpt{},
|
||||||
|
want: []*operations.ListJSONItem{{
|
||||||
|
Path: "file1",
|
||||||
|
Name: "file1",
|
||||||
|
Size: 5,
|
||||||
|
ModTime: operations.Timestamp{When: t1},
|
||||||
|
IsDir: false,
|
||||||
|
}, {
|
||||||
|
Path: "sub",
|
||||||
|
Name: "sub",
|
||||||
|
IsDir: true,
|
||||||
|
}},
|
||||||
|
}, {
|
||||||
|
name: "FilesOnly",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
FilesOnly: true,
|
||||||
|
},
|
||||||
|
want: []*operations.ListJSONItem{{
|
||||||
|
Path: "file1",
|
||||||
|
Name: "file1",
|
||||||
|
Size: 5,
|
||||||
|
ModTime: operations.Timestamp{When: t1},
|
||||||
|
IsDir: false,
|
||||||
|
}},
|
||||||
|
}, {
|
||||||
|
name: "DirsOnly",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
DirsOnly: true,
|
||||||
|
},
|
||||||
|
want: []*operations.ListJSONItem{{
|
||||||
|
Path: "sub",
|
||||||
|
Name: "sub",
|
||||||
|
IsDir: true,
|
||||||
|
}},
|
||||||
|
}, {
|
||||||
|
name: "Recurse",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
Recurse: true,
|
||||||
|
},
|
||||||
|
want: []*operations.ListJSONItem{{
|
||||||
|
Path: "file1",
|
||||||
|
Name: "file1",
|
||||||
|
Size: 5,
|
||||||
|
ModTime: operations.Timestamp{When: t1},
|
||||||
|
IsDir: false,
|
||||||
|
}, {
|
||||||
|
Path: "sub",
|
||||||
|
Name: "sub",
|
||||||
|
IsDir: true,
|
||||||
|
}, {
|
||||||
|
Path: "sub/file2",
|
||||||
|
Name: "file2",
|
||||||
|
Size: 9,
|
||||||
|
ModTime: operations.Timestamp{When: t2},
|
||||||
|
IsDir: false,
|
||||||
|
}},
|
||||||
|
}, {
|
||||||
|
name: "SubDir",
|
||||||
|
remote: "sub",
|
||||||
|
opt: operations.ListJSONOpt{},
|
||||||
|
want: []*operations.ListJSONItem{{
|
||||||
|
Path: "sub/file2",
|
||||||
|
Name: "file2",
|
||||||
|
Size: 9,
|
||||||
|
ModTime: operations.Timestamp{When: t2},
|
||||||
|
IsDir: false,
|
||||||
|
}},
|
||||||
|
}, {
|
||||||
|
name: "NoModTime",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
FilesOnly: true,
|
||||||
|
NoModTime: true,
|
||||||
|
},
|
||||||
|
want: []*operations.ListJSONItem{{
|
||||||
|
Path: "file1",
|
||||||
|
Name: "file1",
|
||||||
|
Size: 5,
|
||||||
|
ModTime: operations.Timestamp{When: time.Time{}},
|
||||||
|
IsDir: false,
|
||||||
|
}},
|
||||||
|
}, {
|
||||||
|
name: "NoModTime",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
FilesOnly: true,
|
||||||
|
NoMimeType: true,
|
||||||
|
},
|
||||||
|
want: []*operations.ListJSONItem{{
|
||||||
|
Path: "file1",
|
||||||
|
Name: "file1",
|
||||||
|
Size: 5,
|
||||||
|
ModTime: operations.Timestamp{When: t1},
|
||||||
|
IsDir: false,
|
||||||
|
}},
|
||||||
|
}, {
|
||||||
|
name: "ShowHash",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
FilesOnly: true,
|
||||||
|
ShowHash: true,
|
||||||
|
},
|
||||||
|
want: []*operations.ListJSONItem{{
|
||||||
|
Path: "file1",
|
||||||
|
Name: "file1",
|
||||||
|
Size: 5,
|
||||||
|
ModTime: operations.Timestamp{When: t1},
|
||||||
|
IsDir: false,
|
||||||
|
}},
|
||||||
|
}, {
|
||||||
|
name: "HashTypes",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
FilesOnly: true,
|
||||||
|
ShowHash: true,
|
||||||
|
HashTypes: []string{"MD5"},
|
||||||
|
},
|
||||||
|
want: []*operations.ListJSONItem{{
|
||||||
|
Path: "file1",
|
||||||
|
Name: "file1",
|
||||||
|
Size: 5,
|
||||||
|
ModTime: operations.Timestamp{When: t1},
|
||||||
|
IsDir: false,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
var got []*operations.ListJSONItem
|
||||||
|
require.NoError(t, operations.ListJSON(ctx, r.Fremote, test.remote, &test.opt, func(item *operations.ListJSONItem) error {
|
||||||
|
got = append(got, item)
|
||||||
|
return nil
|
||||||
|
}))
|
||||||
|
sort.Slice(got, func(i, j int) bool {
|
||||||
|
return got[i].Path < got[j].Path
|
||||||
|
})
|
||||||
|
require.Equal(t, len(test.want), len(got), "Wrong number of results")
|
||||||
|
for i := range test.want {
|
||||||
|
compareListJSONItem(t, test.want[i], got[i], precision)
|
||||||
|
if test.opt.NoMimeType {
|
||||||
|
assert.Equal(t, "", got[i].MimeType)
|
||||||
|
} else {
|
||||||
|
assert.NotEqual(t, "", got[i].MimeType)
|
||||||
|
}
|
||||||
|
if test.opt.ShowHash {
|
||||||
|
hashes := got[i].Hashes
|
||||||
|
assert.NotNil(t, hashes)
|
||||||
|
if len(test.opt.HashTypes) > 0 && len(hashes) > 0 {
|
||||||
|
assert.Equal(t, 1, len(hashes))
|
||||||
|
}
|
||||||
|
if hashes["crc32"] != "" {
|
||||||
|
assert.Equal(t, "9ee760e5", hashes["crc32"])
|
||||||
|
}
|
||||||
|
if hashes["dropbox"] != "" {
|
||||||
|
assert.Equal(t, "f4d62afeaee6f35d3efdd8c66623360395165473bcc958f835343eb3f542f983", hashes["dropbox"])
|
||||||
|
}
|
||||||
|
if hashes["mailru"] != "" {
|
||||||
|
assert.Equal(t, "66696c6531000000000000000000000000000000", hashes["mailru"])
|
||||||
|
}
|
||||||
|
if hashes["md5"] != "" {
|
||||||
|
assert.Equal(t, "826e8142e6baabe8af779f5f490cf5f5", hashes["md5"])
|
||||||
|
}
|
||||||
|
if hashes["quickxor"] != "" {
|
||||||
|
assert.Equal(t, "6648031bca100300000000000500000000000000", hashes["quickxor"])
|
||||||
|
}
|
||||||
|
if hashes["sha1"] != "" {
|
||||||
|
assert.Equal(t, "60b27f004e454aca81b0480209cce5081ec52390", hashes["sha1"])
|
||||||
|
}
|
||||||
|
if hashes["sha256"] != "" {
|
||||||
|
assert.Equal(t, "c147efcfc2d7ea666a9e4f5187b115c90903f0fc896a56df9a6ef5d8f3fc9f31", hashes["sha256"])
|
||||||
|
}
|
||||||
|
if hashes["whirlpool"] != "" {
|
||||||
|
assert.Equal(t, "02fa11755b6470bfc5aab6d94cde5cf2939474fb5b0ebbf8ddf3d32bf06aa438eb92eac097047c02017dc1c317ee83fa8a2717ca4d544b4ee75b3231d1c466b0", hashes["whirlpool"])
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
assert.Nil(t, got[i].Hashes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatJSON(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
r := fstest.NewRun(t)
|
||||||
|
defer r.Finalise()
|
||||||
|
file1 := r.WriteBoth(ctx, "file1", "file1", t1)
|
||||||
|
file2 := r.WriteBoth(ctx, "sub/file2", "sub/file2", t2)
|
||||||
|
|
||||||
|
fstest.CheckItems(t, r.Fremote, file1, file2)
|
||||||
|
precision := fs.GetModifyWindow(ctx, r.Fremote)
|
||||||
|
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
remote string
|
||||||
|
opt operations.ListJSONOpt
|
||||||
|
want *operations.ListJSONItem
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Root",
|
||||||
|
remote: "",
|
||||||
|
opt: operations.ListJSONOpt{},
|
||||||
|
want: &operations.ListJSONItem{
|
||||||
|
Path: "",
|
||||||
|
Name: "",
|
||||||
|
IsDir: true,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
name: "Dir",
|
||||||
|
remote: "sub",
|
||||||
|
opt: operations.ListJSONOpt{},
|
||||||
|
want: &operations.ListJSONItem{
|
||||||
|
Path: "sub",
|
||||||
|
Name: "sub",
|
||||||
|
IsDir: true,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
name: "File",
|
||||||
|
remote: "file1",
|
||||||
|
opt: operations.ListJSONOpt{},
|
||||||
|
want: &operations.ListJSONItem{
|
||||||
|
Path: "file1",
|
||||||
|
Name: "file1",
|
||||||
|
Size: 5,
|
||||||
|
ModTime: operations.Timestamp{When: t1},
|
||||||
|
IsDir: false,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
name: "NotFound",
|
||||||
|
remote: "notfound",
|
||||||
|
opt: operations.ListJSONOpt{},
|
||||||
|
want: nil,
|
||||||
|
}, {
|
||||||
|
name: "DirFilesOnly",
|
||||||
|
remote: "sub",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
FilesOnly: true,
|
||||||
|
},
|
||||||
|
want: nil,
|
||||||
|
}, {
|
||||||
|
name: "FileFilesOnly",
|
||||||
|
remote: "file1",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
FilesOnly: true,
|
||||||
|
},
|
||||||
|
want: &operations.ListJSONItem{
|
||||||
|
Path: "file1",
|
||||||
|
Name: "file1",
|
||||||
|
Size: 5,
|
||||||
|
ModTime: operations.Timestamp{When: t1},
|
||||||
|
IsDir: false,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
name: "NotFoundFilesOnly",
|
||||||
|
remote: "notfound",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
FilesOnly: true,
|
||||||
|
},
|
||||||
|
want: nil,
|
||||||
|
}, {
|
||||||
|
name: "DirDirsOnly",
|
||||||
|
remote: "sub",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
DirsOnly: true,
|
||||||
|
},
|
||||||
|
want: &operations.ListJSONItem{
|
||||||
|
Path: "sub",
|
||||||
|
Name: "sub",
|
||||||
|
IsDir: true,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
name: "FileDirsOnly",
|
||||||
|
remote: "file1",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
DirsOnly: true,
|
||||||
|
},
|
||||||
|
want: nil,
|
||||||
|
}, {
|
||||||
|
name: "NotFoundDirsOnly",
|
||||||
|
remote: "notfound",
|
||||||
|
opt: operations.ListJSONOpt{
|
||||||
|
DirsOnly: true,
|
||||||
|
},
|
||||||
|
want: nil,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
got, err := operations.StatJSON(ctx, r.Fremote, test.remote, &test.opt)
|
||||||
|
require.NoError(t, err)
|
||||||
|
if test.want == nil {
|
||||||
|
assert.Nil(t, got)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
require.NotNil(t, got)
|
||||||
|
compareListJSONItem(t, test.want, got, precision)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("RootNotFound", func(t *testing.T) {
|
||||||
|
f, err := fs.NewFs(ctx, r.FremoteName+"/notfound")
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = operations.StatJSON(ctx, f, "", &operations.ListJSONOpt{})
|
||||||
|
// This should return an error except for bucket based remotes
|
||||||
|
assert.True(t, err != nil || f.Features().BucketBased, "Need an error for non bucket based backends")
|
||||||
|
})
|
||||||
|
}
|
|
@ -31,6 +31,10 @@ func init() {
|
||||||
- showEncrypted - If set show decrypted names
|
- showEncrypted - If set show decrypted names
|
||||||
- showOrigIDs - If set show the IDs for each item if known
|
- showOrigIDs - If set show the IDs for each item if known
|
||||||
- showHash - If set return a dictionary of hashes
|
- showHash - If set return a dictionary of hashes
|
||||||
|
- noMimeType - If set don't show mime types
|
||||||
|
- dirsOnly - If set only show directories
|
||||||
|
- filesOnly - If set only show files
|
||||||
|
- hashTypes - array of strings of hash types to show if showHash set
|
||||||
|
|
||||||
The result is
|
The result is
|
||||||
|
|
||||||
|
@ -66,6 +70,51 @@ func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rc.Add(rc.Call{
|
||||||
|
Path: "operations/stat",
|
||||||
|
AuthRequired: true,
|
||||||
|
Fn: rcStat,
|
||||||
|
Title: "Give information about the supplied file or directory",
|
||||||
|
Help: `This takes the following parameters
|
||||||
|
|
||||||
|
- fs - a remote name string eg "drive:"
|
||||||
|
- remote - a path within that remote eg "dir"
|
||||||
|
- opt - a dictionary of options to control the listing (optional)
|
||||||
|
- see operations/list for the options
|
||||||
|
|
||||||
|
The result is
|
||||||
|
|
||||||
|
- item - an object as described in the lsjson command. Will be null if not found.
|
||||||
|
|
||||||
|
Note that if you are only interested in files then it is much more
|
||||||
|
efficient to set the filesOnly flag in the options.
|
||||||
|
|
||||||
|
See the [lsjson command](/commands/rclone_lsjson/) for more information on the above and examples.
|
||||||
|
`,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// List the directory
|
||||||
|
func rcStat(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
||||||
|
f, remote, err := rc.GetFsAndRemote(ctx, in)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var opt ListJSONOpt
|
||||||
|
err = in.GetStruct("opt", &opt)
|
||||||
|
if rc.NotErrParamNotFound(err) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
item, err := StatJSON(ctx, f, remote, &opt)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = make(rc.Params)
|
||||||
|
out["item"] = item
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
rc.Add(rc.Call{
|
rc.Add(rc.Call{
|
||||||
Path: "operations/about",
|
Path: "operations/about",
|
||||||
|
|
|
@ -261,6 +261,59 @@ func TestRcList(t *testing.T) {
|
||||||
checkFile2(list[2])
|
checkFile2(list[2])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// operations/stat: Stat the given remote and path in JSON format.
|
||||||
|
func TestRcStat(t *testing.T) {
|
||||||
|
r, call := rcNewRun(t, "operations/stat")
|
||||||
|
defer r.Finalise()
|
||||||
|
|
||||||
|
file1 := r.WriteObject(context.Background(), "subdir/a", "a", t1)
|
||||||
|
|
||||||
|
fstest.CheckItems(t, r.Fremote, file1)
|
||||||
|
|
||||||
|
fetch := func(t *testing.T, remotePath string) *operations.ListJSONItem {
|
||||||
|
in := rc.Params{
|
||||||
|
"fs": r.FremoteName,
|
||||||
|
"remote": remotePath,
|
||||||
|
}
|
||||||
|
out, err := call.Fn(context.Background(), in)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return out["item"].(*operations.ListJSONItem)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("Root", func(t *testing.T) {
|
||||||
|
stat := fetch(t, "")
|
||||||
|
assert.Equal(t, "", stat.Path)
|
||||||
|
assert.Equal(t, "", stat.Name)
|
||||||
|
assert.Equal(t, int64(-1), stat.Size)
|
||||||
|
assert.Equal(t, "inode/directory", stat.MimeType)
|
||||||
|
assert.Equal(t, true, stat.IsDir)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("File", func(t *testing.T) {
|
||||||
|
stat := fetch(t, "subdir/a")
|
||||||
|
assert.WithinDuration(t, t1, stat.ModTime.When, time.Second)
|
||||||
|
assert.Equal(t, "subdir/a", stat.Path)
|
||||||
|
assert.Equal(t, "a", stat.Name)
|
||||||
|
assert.Equal(t, int64(1), stat.Size)
|
||||||
|
assert.Equal(t, "application/octet-stream", stat.MimeType)
|
||||||
|
assert.Equal(t, false, stat.IsDir)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("Subdir", func(t *testing.T) {
|
||||||
|
stat := fetch(t, "subdir")
|
||||||
|
assert.Equal(t, "subdir", stat.Path)
|
||||||
|
assert.Equal(t, "subdir", stat.Name)
|
||||||
|
assert.Equal(t, int64(-1), stat.Size)
|
||||||
|
assert.Equal(t, "inode/directory", stat.MimeType)
|
||||||
|
assert.Equal(t, true, stat.IsDir)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("NotFound", func(t *testing.T) {
|
||||||
|
stat := fetch(t, "notfound")
|
||||||
|
assert.Nil(t, stat)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// operations/mkdir: Make a destination directory or container
|
// operations/mkdir: Make a destination directory or container
|
||||||
func TestRcMkdir(t *testing.T) {
|
func TestRcMkdir(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
Loading…
Reference in a new issue