Remove backend dependency from fs/hash
This commit is contained in:
parent
def411da62
commit
1dc8bcd48c
13 changed files with 138 additions and 150 deletions
|
@ -39,6 +39,7 @@ import (
|
||||||
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/team"
|
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/team"
|
||||||
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
|
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
"github.com/rclone/rclone/backend/dropbox/dbhash"
|
||||||
"github.com/rclone/rclone/fs"
|
"github.com/rclone/rclone/fs"
|
||||||
"github.com/rclone/rclone/fs/config"
|
"github.com/rclone/rclone/fs/config"
|
||||||
"github.com/rclone/rclone/fs/config/configmap"
|
"github.com/rclone/rclone/fs/config/configmap"
|
||||||
|
@ -105,10 +106,14 @@ var (
|
||||||
// A regexp matching path names for files Dropbox ignores
|
// A regexp matching path names for files Dropbox ignores
|
||||||
// See https://www.dropbox.com/en/help/145 - Ignored files
|
// See https://www.dropbox.com/en/help/145 - Ignored files
|
||||||
ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`)
|
ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`)
|
||||||
|
|
||||||
|
// DbHashType is the hash.Type for Dropbox
|
||||||
|
DbHashType hash.Type
|
||||||
)
|
)
|
||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
|
DbHashType = hash.RegisterHash("Dropbox", 64, dbhash.New)
|
||||||
fs.Register(&fs.RegInfo{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "dropbox",
|
Name: "dropbox",
|
||||||
Description: "Dropbox",
|
Description: "Dropbox",
|
||||||
|
@ -881,7 +886,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||||
|
|
||||||
// Hashes returns the supported hash sets.
|
// Hashes returns the supported hash sets.
|
||||||
func (f *Fs) Hashes() hash.Set {
|
func (f *Fs) Hashes() hash.Set {
|
||||||
return hash.Set(hash.Dropbox)
|
return hash.Set(DbHashType)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
@ -906,7 +911,7 @@ func (o *Object) Remote() string {
|
||||||
|
|
||||||
// Hash returns the dropbox special hash
|
// Hash returns the dropbox special hash
|
||||||
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||||
if t != hash.Dropbox {
|
if t != DbHashType {
|
||||||
return "", hash.ErrUnsupported
|
return "", hash.ErrUnsupported
|
||||||
}
|
}
|
||||||
err := o.readMetaData()
|
err := o.readMetaData()
|
||||||
|
|
|
@ -686,7 +686,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||||
|
|
||||||
// Hashes returns the supported hash sets.
|
// Hashes returns the supported hash sets.
|
||||||
func (f *Fs) Hashes() hash.Set {
|
func (f *Fs) Hashes() hash.Set {
|
||||||
return hash.Supported
|
return hash.Supported()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
|
@ -44,7 +44,7 @@ func TestUpdatingCheck(t *testing.T) {
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
o := &Object{size: fi.Size(), modTime: fi.ModTime(), fs: &Fs{}}
|
o := &Object{size: fi.Size(), modTime: fi.ModTime(), fs: &Fs{}}
|
||||||
wrappedFd := readers.NewLimitedReadCloser(fd, -1)
|
wrappedFd := readers.NewLimitedReadCloser(fd, -1)
|
||||||
hash, err := hash.NewMultiHasherTypes(hash.Supported)
|
hash, err := hash.NewMultiHasherTypes(hash.Supported())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
in := localOpenFile{
|
in := localOpenFile{
|
||||||
o: o,
|
o: o,
|
||||||
|
|
|
@ -59,6 +59,9 @@ var (
|
||||||
ErrorDirAlreadyExists = errors.New("directory already exists")
|
ErrorDirAlreadyExists = errors.New("directory already exists")
|
||||||
ErrorDirSourceNotExists = errors.New("directory source does not exist")
|
ErrorDirSourceNotExists = errors.New("directory source does not exist")
|
||||||
ErrorInvalidName = errors.New("invalid characters in object name")
|
ErrorInvalidName = errors.New("invalid characters in object name")
|
||||||
|
|
||||||
|
// MrHashType is the hash.Type for Mailru
|
||||||
|
MrHashType hash.Type
|
||||||
)
|
)
|
||||||
|
|
||||||
// Description of how to authorize
|
// Description of how to authorize
|
||||||
|
@ -74,6 +77,7 @@ var oauthConfig = &oauth2.Config{
|
||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
|
MrHashType = hash.RegisterHash("MailruHash", 40, mrhash.New)
|
||||||
fs.Register(&fs.RegInfo{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "mailru",
|
Name: "mailru",
|
||||||
Description: "Mail.ru Cloud",
|
Description: "Mail.ru Cloud",
|
||||||
|
@ -1591,7 +1595,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||||
// Skip an extra speedup request if file fits in hash.
|
// Skip an extra speedup request if file fits in hash.
|
||||||
if size > mrhash.Size {
|
if size > mrhash.Size {
|
||||||
// Request hash from source.
|
// Request hash from source.
|
||||||
if srcHash, err := src.Hash(ctx, hash.Mailru); err == nil && srcHash != "" {
|
if srcHash, err := src.Hash(ctx, MrHashType); err == nil && srcHash != "" {
|
||||||
fileHash, _ = mrhash.DecodeString(srcHash)
|
fileHash, _ = mrhash.DecodeString(srcHash)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1762,7 +1766,7 @@ func makeTempFile(ctx context.Context, tmpFs fs.Fs, wrapIn io.Reader, src fs.Obj
|
||||||
hashType := hash.SHA1
|
hashType := hash.SHA1
|
||||||
|
|
||||||
// Calculate Mailru and spool verification hashes in transit
|
// Calculate Mailru and spool verification hashes in transit
|
||||||
hashSet := hash.NewHashSet(hash.Mailru, hashType)
|
hashSet := hash.NewHashSet(MrHashType, hashType)
|
||||||
hasher, err := hash.NewMultiHasherTypes(hashSet)
|
hasher, err := hash.NewMultiHasherTypes(hashSet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
|
@ -1784,7 +1788,7 @@ func makeTempFile(ctx context.Context, tmpFs fs.Fs, wrapIn io.Reader, src fs.Obj
|
||||||
return nil, nil, mrhash.ErrorInvalidHash
|
return nil, nil, mrhash.ErrorInvalidHash
|
||||||
}
|
}
|
||||||
|
|
||||||
mrHash, err = mrhash.DecodeString(sums[hash.Mailru])
|
mrHash, err = mrhash.DecodeString(sums[MrHashType])
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1972,7 +1976,7 @@ func (o *Object) Size() int64 {
|
||||||
// Hash returns the MD5 or SHA1 sum of an object
|
// Hash returns the MD5 or SHA1 sum of an object
|
||||||
// returning a lowercase hex string
|
// returning a lowercase hex string
|
||||||
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||||
if t == hash.Mailru {
|
if t == MrHashType {
|
||||||
return hex.EncodeToString(o.mrHash), nil
|
return hex.EncodeToString(o.mrHash), nil
|
||||||
}
|
}
|
||||||
return "", hash.ErrUnsupported
|
return "", hash.ErrUnsupported
|
||||||
|
@ -2354,7 +2358,7 @@ func (f *Fs) Precision() time.Duration {
|
||||||
|
|
||||||
// Hashes returns the supported hash sets
|
// Hashes returns the supported hash sets
|
||||||
func (f *Fs) Hashes() hash.Set {
|
func (f *Fs) Hashes() hash.Set {
|
||||||
return hash.Set(hash.Mailru)
|
return hash.Set(MrHashType)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Features returns the optional features of this Fs
|
// Features returns the optional features of this Fs
|
||||||
|
|
|
@ -17,6 +17,7 @@ import (
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/rclone/rclone/backend/onedrive/api"
|
"github.com/rclone/rclone/backend/onedrive/api"
|
||||||
|
"github.com/rclone/rclone/backend/onedrive/quickxorhash"
|
||||||
"github.com/rclone/rclone/fs"
|
"github.com/rclone/rclone/fs"
|
||||||
"github.com/rclone/rclone/fs/config"
|
"github.com/rclone/rclone/fs/config"
|
||||||
"github.com/rclone/rclone/fs/config/configmap"
|
"github.com/rclone/rclone/fs/config/configmap"
|
||||||
|
@ -65,10 +66,14 @@ var (
|
||||||
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
|
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
|
||||||
RedirectURL: oauthutil.RedirectLocalhostURL,
|
RedirectURL: oauthutil.RedirectLocalhostURL,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// QuickXorHashType is the hash.Type for OneDrive
|
||||||
|
QuickXorHashType hash.Type
|
||||||
)
|
)
|
||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
|
QuickXorHashType = hash.RegisterHash("QuickXorHash", 40, quickxorhash.New)
|
||||||
fs.Register(&fs.RegInfo{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "onedrive",
|
Name: "onedrive",
|
||||||
Description: "Microsoft OneDrive",
|
Description: "Microsoft OneDrive",
|
||||||
|
@ -1194,7 +1199,7 @@ func (f *Fs) Hashes() hash.Set {
|
||||||
if f.driveType == driveTypePersonal {
|
if f.driveType == driveTypePersonal {
|
||||||
return hash.Set(hash.SHA1)
|
return hash.Set(hash.SHA1)
|
||||||
}
|
}
|
||||||
return hash.Set(hash.QuickXorHash)
|
return hash.Set(QuickXorHashType)
|
||||||
}
|
}
|
||||||
|
|
||||||
// PublicLink returns a link for downloading without accout.
|
// PublicLink returns a link for downloading without accout.
|
||||||
|
@ -1270,7 +1275,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||||
return o.sha1, nil
|
return o.sha1, nil
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if t == hash.QuickXorHash {
|
if t == QuickXorHashType {
|
||||||
return o.quickxorhash, nil
|
return o.quickxorhash, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,9 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"github.com/rclone/rclone/backend/dropbox/dbhash"
|
||||||
"github.com/rclone/rclone/cmd"
|
"github.com/rclone/rclone/cmd"
|
||||||
|
"github.com/rclone/rclone/fs/hash"
|
||||||
"github.com/rclone/rclone/fs/operations"
|
"github.com/rclone/rclone/fs/operations"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
@ -26,7 +28,8 @@ The output is in the same format as md5sum and sha1sum.
|
||||||
cmd.CheckArgs(1, 1, command, args)
|
cmd.CheckArgs(1, 1, command, args)
|
||||||
fsrc := cmd.NewFsSrc(args)
|
fsrc := cmd.NewFsSrc(args)
|
||||||
cmd.Run(false, false, command, func() error {
|
cmd.Run(false, false, command, func() error {
|
||||||
return operations.DropboxHashSum(context.Background(), fsrc, os.Stdout)
|
dbHashType := hash.RegisterHash("Dropbox", 64, dbhash.New)
|
||||||
|
return operations.HashLister(context.Background(), dbHashType, fsrc, os.Stdout)
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,7 +41,7 @@ Then
|
||||||
cmd.CheckArgs(0, 2, command, args)
|
cmd.CheckArgs(0, 2, command, args)
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
fmt.Printf("Supported hashes are:\n")
|
fmt.Printf("Supported hashes are:\n")
|
||||||
for _, ht := range hash.Supported.Array() {
|
for _, ht := range hash.Supported().Array() {
|
||||||
fmt.Printf(" * %v\n", ht)
|
fmt.Printf(" * %v\n", ht)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|
178
fs/hash/hash.go
178
fs/hash/hash.go
|
@ -12,64 +12,88 @@ import (
|
||||||
|
|
||||||
"github.com/jzelinskie/whirlpool"
|
"github.com/jzelinskie/whirlpool"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/rclone/rclone/backend/dropbox/dbhash"
|
|
||||||
"github.com/rclone/rclone/backend/mailru/mrhash"
|
|
||||||
"github.com/rclone/rclone/backend/onedrive/quickxorhash"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Type indicates a standard hashing algorithm
|
// Type indicates a standard hashing algorithm
|
||||||
type Type int
|
type Type int
|
||||||
|
|
||||||
|
type hashDefinition struct {
|
||||||
|
width int
|
||||||
|
name string
|
||||||
|
newFunc func() hash.Hash
|
||||||
|
hashType Type
|
||||||
|
}
|
||||||
|
|
||||||
|
var hashes []*hashDefinition
|
||||||
|
var highestType Type = 1
|
||||||
|
|
||||||
|
// RegisterHash adds a new Hash to the list and returns it Type
|
||||||
|
func RegisterHash(name string, width int, newFunc func() hash.Hash) Type {
|
||||||
|
definition := &hashDefinition{
|
||||||
|
name: name,
|
||||||
|
width: width,
|
||||||
|
newFunc: newFunc,
|
||||||
|
hashType: highestType,
|
||||||
|
}
|
||||||
|
hashes = append(hashes, definition)
|
||||||
|
highestType = highestType << 1
|
||||||
|
|
||||||
|
return definition.hashType
|
||||||
|
}
|
||||||
|
|
||||||
// ErrUnsupported should be returned by filesystem,
|
// ErrUnsupported should be returned by filesystem,
|
||||||
// if it is requested to deliver an unsupported hash type.
|
// if it is requested to deliver an unsupported hash type.
|
||||||
var ErrUnsupported = errors.New("hash type not supported")
|
var ErrUnsupported = errors.New("hash type not supported")
|
||||||
|
|
||||||
const (
|
var (
|
||||||
|
// None indicates no hashes are supported
|
||||||
|
None Type
|
||||||
|
|
||||||
// MD5 indicates MD5 support
|
// MD5 indicates MD5 support
|
||||||
MD5 Type = 1 << iota
|
MD5 Type
|
||||||
|
|
||||||
// SHA1 indicates SHA-1 support
|
// SHA1 indicates SHA-1 support
|
||||||
SHA1
|
SHA1 Type
|
||||||
|
|
||||||
// Dropbox indicates Dropbox special hash
|
|
||||||
// https://www.dropbox.com/developers/reference/content-hash
|
|
||||||
Dropbox
|
|
||||||
|
|
||||||
// QuickXorHash indicates Microsoft onedrive hash
|
|
||||||
// https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
|
|
||||||
QuickXorHash
|
|
||||||
|
|
||||||
// Whirlpool indicates Whirlpool support
|
// Whirlpool indicates Whirlpool support
|
||||||
Whirlpool
|
Whirlpool Type
|
||||||
|
|
||||||
// CRC32 indicates CRC-32 support
|
// CRC32 indicates CRC-32 support
|
||||||
CRC32
|
CRC32 Type
|
||||||
|
|
||||||
// Mailru indicates Mailru special hash
|
|
||||||
Mailru
|
|
||||||
|
|
||||||
// None indicates no hashes are supported
|
|
||||||
None Type = 0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
MD5 = RegisterHash("MD5", 32, md5.New)
|
||||||
|
SHA1 = RegisterHash("SHA-1", 40, sha1.New)
|
||||||
|
Whirlpool = RegisterHash("Whirlpool", 128, whirlpool.New)
|
||||||
|
CRC32 = RegisterHash("CRC32", 8, func() hash.Hash { return crc32.NewIEEE() })
|
||||||
|
}
|
||||||
|
|
||||||
// Supported returns a set of all the supported hashes by
|
// Supported returns a set of all the supported hashes by
|
||||||
// HashStream and MultiHasher.
|
// HashStream and MultiHasher.
|
||||||
var Supported = NewHashSet(MD5, SHA1, Dropbox, QuickXorHash, Whirlpool, CRC32, Mailru)
|
func Supported() Set {
|
||||||
|
var types []Type
|
||||||
|
for _, v := range hashes {
|
||||||
|
types = append(types, v.hashType)
|
||||||
|
}
|
||||||
|
|
||||||
|
return NewHashSet(types...)
|
||||||
|
}
|
||||||
|
|
||||||
// Width returns the width in characters for any HashType
|
// Width returns the width in characters for any HashType
|
||||||
var Width = map[Type]int{
|
func Width(hashType Type) int {
|
||||||
MD5: 32,
|
for _, v := range hashes {
|
||||||
SHA1: 40,
|
if v.hashType == hashType {
|
||||||
Dropbox: 64,
|
return v.width
|
||||||
QuickXorHash: 40,
|
}
|
||||||
Whirlpool: 128,
|
}
|
||||||
CRC32: 8,
|
|
||||||
Mailru: 40,
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream will calculate hashes of all supported hash types.
|
// Stream will calculate hashes of all supported hash types.
|
||||||
func Stream(r io.Reader) (map[Type]string, error) {
|
func Stream(r io.Reader) (map[Type]string, error) {
|
||||||
return StreamTypes(r, Supported)
|
return StreamTypes(r, Supported())
|
||||||
}
|
}
|
||||||
|
|
||||||
// StreamTypes will calculate hashes of the requested hash types.
|
// StreamTypes will calculate hashes of the requested hash types.
|
||||||
|
@ -93,52 +117,34 @@ func StreamTypes(r io.Reader, set Set) (map[Type]string, error) {
|
||||||
// String returns a string representation of the hash type.
|
// String returns a string representation of the hash type.
|
||||||
// The function will panic if the hash type is unknown.
|
// The function will panic if the hash type is unknown.
|
||||||
func (h Type) String() string {
|
func (h Type) String() string {
|
||||||
switch h {
|
if h == None {
|
||||||
case None:
|
|
||||||
return "None"
|
return "None"
|
||||||
case MD5:
|
|
||||||
return "MD5"
|
|
||||||
case SHA1:
|
|
||||||
return "SHA-1"
|
|
||||||
case Dropbox:
|
|
||||||
return "DropboxHash"
|
|
||||||
case QuickXorHash:
|
|
||||||
return "QuickXorHash"
|
|
||||||
case Whirlpool:
|
|
||||||
return "Whirlpool"
|
|
||||||
case CRC32:
|
|
||||||
return "CRC-32"
|
|
||||||
case Mailru:
|
|
||||||
return "MailruHash"
|
|
||||||
default:
|
|
||||||
err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h))
|
|
||||||
panic(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, v := range hashes {
|
||||||
|
if v.hashType == h {
|
||||||
|
return v.name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h))
|
||||||
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set a Type from a flag
|
// Set a Type from a flag
|
||||||
func (h *Type) Set(s string) error {
|
func (h *Type) Set(s string) error {
|
||||||
switch s {
|
if s == "None" {
|
||||||
case "None":
|
|
||||||
*h = None
|
*h = None
|
||||||
case "MD5":
|
|
||||||
*h = MD5
|
|
||||||
case "SHA-1":
|
|
||||||
*h = SHA1
|
|
||||||
case "DropboxHash":
|
|
||||||
*h = Dropbox
|
|
||||||
case "QuickXorHash":
|
|
||||||
*h = QuickXorHash
|
|
||||||
case "Whirlpool":
|
|
||||||
*h = Whirlpool
|
|
||||||
case "CRC-32":
|
|
||||||
*h = CRC32
|
|
||||||
case "MailruHash":
|
|
||||||
*h = Mailru
|
|
||||||
default:
|
|
||||||
return errors.Errorf("Unknown hash type %q", s)
|
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
|
for _, v := range hashes {
|
||||||
|
if v.name == s {
|
||||||
|
*h = v.hashType
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return errors.Errorf("Unknown hash type %q", s)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Type of the value
|
// Type of the value
|
||||||
|
@ -150,32 +156,28 @@ func (h Type) Type() string {
|
||||||
// The types must be a subset of SupportedHashes,
|
// The types must be a subset of SupportedHashes,
|
||||||
// and this function must support all types.
|
// and this function must support all types.
|
||||||
func fromTypes(set Set) (map[Type]hash.Hash, error) {
|
func fromTypes(set Set) (map[Type]hash.Hash, error) {
|
||||||
if !set.SubsetOf(Supported) {
|
if !set.SubsetOf(Supported()) {
|
||||||
return nil, errors.Errorf("requested set %08x contains unknown hash types", int(set))
|
return nil, errors.Errorf("requested set %08x contains unknown hash types", int(set))
|
||||||
}
|
}
|
||||||
var hashers = make(map[Type]hash.Hash)
|
var hashers = make(map[Type]hash.Hash)
|
||||||
|
|
||||||
types := set.Array()
|
types := set.Array()
|
||||||
for _, t := range types {
|
for _, t := range types {
|
||||||
switch t {
|
for _, v := range hashes {
|
||||||
case MD5:
|
if t != v.hashType {
|
||||||
hashers[t] = md5.New()
|
continue
|
||||||
case SHA1:
|
}
|
||||||
hashers[t] = sha1.New()
|
|
||||||
case Dropbox:
|
hashers[t] = v.newFunc()
|
||||||
hashers[t] = dbhash.New()
|
break
|
||||||
case QuickXorHash:
|
}
|
||||||
hashers[t] = quickxorhash.New()
|
|
||||||
case Whirlpool:
|
if hashers[t] == nil {
|
||||||
hashers[t] = whirlpool.New()
|
|
||||||
case CRC32:
|
|
||||||
hashers[t] = crc32.NewIEEE()
|
|
||||||
case Mailru:
|
|
||||||
hashers[t] = mrhash.New()
|
|
||||||
default:
|
|
||||||
err := fmt.Sprintf("internal error: Unsupported hash type %v", t)
|
err := fmt.Sprintf("internal error: Unsupported hash type %v", t)
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return hashers, nil
|
return hashers, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -202,7 +204,7 @@ type MultiHasher struct {
|
||||||
// NewMultiHasher will return a hash writer that will write all
|
// NewMultiHasher will return a hash writer that will write all
|
||||||
// supported hash types.
|
// supported hash types.
|
||||||
func NewMultiHasher() *MultiHasher {
|
func NewMultiHasher() *MultiHasher {
|
||||||
h, err := NewMultiHasherTypes(Supported)
|
h, err := NewMultiHasherTypes(Supported())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic("internal error: could not create multihasher")
|
panic("internal error: could not create multihasher")
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ package hash_test
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"io"
|
"io"
|
||||||
|
"log"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/rclone/rclone/fs/hash"
|
"github.com/rclone/rclone/fs/hash"
|
||||||
|
@ -23,6 +24,7 @@ func TestHashSet(t *testing.T) {
|
||||||
assert.Len(t, a, 0)
|
assert.Len(t, a, 0)
|
||||||
|
|
||||||
h = h.Add(hash.MD5)
|
h = h.Add(hash.MD5)
|
||||||
|
log.Println(h)
|
||||||
assert.Equal(t, 1, h.Count())
|
assert.Equal(t, 1, h.Count())
|
||||||
assert.Equal(t, hash.MD5, h.GetOne())
|
assert.Equal(t, hash.MD5, h.GetOne())
|
||||||
a = h.Array()
|
a = h.Array()
|
||||||
|
@ -30,10 +32,10 @@ func TestHashSet(t *testing.T) {
|
||||||
assert.Equal(t, a[0], hash.MD5)
|
assert.Equal(t, a[0], hash.MD5)
|
||||||
|
|
||||||
// Test overlap, with all hashes
|
// Test overlap, with all hashes
|
||||||
h = h.Overlap(hash.Supported)
|
h = h.Overlap(hash.Supported())
|
||||||
assert.Equal(t, 1, h.Count())
|
assert.Equal(t, 1, h.Count())
|
||||||
assert.Equal(t, hash.MD5, h.GetOne())
|
assert.Equal(t, hash.MD5, h.GetOne())
|
||||||
assert.True(t, h.SubsetOf(hash.Supported))
|
assert.True(t, h.SubsetOf(hash.Supported()))
|
||||||
assert.True(t, h.SubsetOf(hash.NewHashSet(hash.MD5)))
|
assert.True(t, h.SubsetOf(hash.NewHashSet(hash.MD5)))
|
||||||
|
|
||||||
h = h.Add(hash.SHA1)
|
h = h.Add(hash.SHA1)
|
||||||
|
@ -42,7 +44,7 @@ func TestHashSet(t *testing.T) {
|
||||||
if !(one == hash.MD5 || one == hash.SHA1) {
|
if !(one == hash.MD5 || one == hash.SHA1) {
|
||||||
t.Fatalf("expected to be either MD5 or SHA1, got %v", one)
|
t.Fatalf("expected to be either MD5 or SHA1, got %v", one)
|
||||||
}
|
}
|
||||||
assert.True(t, h.SubsetOf(hash.Supported))
|
assert.True(t, h.SubsetOf(hash.Supported()))
|
||||||
assert.False(t, h.SubsetOf(hash.NewHashSet(hash.MD5)))
|
assert.False(t, h.SubsetOf(hash.NewHashSet(hash.MD5)))
|
||||||
assert.False(t, h.SubsetOf(hash.NewHashSet(hash.SHA1)))
|
assert.False(t, h.SubsetOf(hash.NewHashSet(hash.SHA1)))
|
||||||
assert.True(t, h.SubsetOf(hash.NewHashSet(hash.MD5, hash.SHA1)))
|
assert.True(t, h.SubsetOf(hash.NewHashSet(hash.MD5, hash.SHA1)))
|
||||||
|
@ -69,26 +71,20 @@ var hashTestSet = []hashTest{
|
||||||
{
|
{
|
||||||
input: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14},
|
input: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14},
|
||||||
output: map[hash.Type]string{
|
output: map[hash.Type]string{
|
||||||
hash.MD5: "bf13fc19e5151ac57d4252e0e0f87abe",
|
hash.MD5: "bf13fc19e5151ac57d4252e0e0f87abe",
|
||||||
hash.SHA1: "3ab6543c08a75f292a5ecedac87ec41642d12166",
|
hash.SHA1: "3ab6543c08a75f292a5ecedac87ec41642d12166",
|
||||||
hash.Dropbox: "214d2fcf3566e94c99ad2f59bd993daca46d8521a0c447adf4b324f53fddc0c7",
|
hash.Whirlpool: "eddf52133d4566d763f716e853d6e4efbabd29e2c2e63f56747b1596172851d34c2df9944beb6640dbdbe3d9b4eb61180720a79e3d15baff31c91e43d63869a4",
|
||||||
hash.QuickXorHash: "0110c000085000031c0001095ec00218d0000700",
|
hash.CRC32: "a6041d7e",
|
||||||
hash.Whirlpool: "eddf52133d4566d763f716e853d6e4efbabd29e2c2e63f56747b1596172851d34c2df9944beb6640dbdbe3d9b4eb61180720a79e3d15baff31c91e43d63869a4",
|
|
||||||
hash.CRC32: "a6041d7e",
|
|
||||||
hash.Mailru: "0102030405060708090a0b0c0d0e000000000000",
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
// Empty data set
|
// Empty data set
|
||||||
{
|
{
|
||||||
input: []byte{},
|
input: []byte{},
|
||||||
output: map[hash.Type]string{
|
output: map[hash.Type]string{
|
||||||
hash.MD5: "d41d8cd98f00b204e9800998ecf8427e",
|
hash.MD5: "d41d8cd98f00b204e9800998ecf8427e",
|
||||||
hash.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
hash.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
||||||
hash.Dropbox: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
hash.Whirlpool: "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3",
|
||||||
hash.QuickXorHash: "0000000000000000000000000000000000000000",
|
hash.CRC32: "00000000",
|
||||||
hash.Whirlpool: "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3",
|
|
||||||
hash.CRC32: "00000000",
|
|
||||||
hash.Mailru: "0000000000000000000000000000000000000000",
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -159,8 +155,8 @@ func TestHashStreamTypes(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHashSetStringer(t *testing.T) {
|
func TestHashSetStringer(t *testing.T) {
|
||||||
h := hash.NewHashSet(hash.SHA1, hash.MD5, hash.Dropbox, hash.QuickXorHash)
|
h := hash.NewHashSet(hash.SHA1, hash.MD5)
|
||||||
assert.Equal(t, h.String(), "[MD5, SHA-1, DropboxHash, QuickXorHash]")
|
assert.Equal(t, h.String(), "[MD5, SHA-1]")
|
||||||
h = hash.NewHashSet(hash.SHA1)
|
h = hash.NewHashSet(hash.SHA1)
|
||||||
assert.Equal(t, h.String(), "[SHA-1]")
|
assert.Equal(t, h.String(), "[SHA-1]")
|
||||||
h = hash.NewHashSet()
|
h = hash.NewHashSet()
|
||||||
|
|
|
@ -79,7 +79,7 @@ func (memoryFs) String() string { return "memory" }
|
||||||
func (memoryFs) Precision() time.Duration { return time.Nanosecond }
|
func (memoryFs) Precision() time.Duration { return time.Nanosecond }
|
||||||
|
|
||||||
// Returns the supported hash types of the filesystem
|
// Returns the supported hash types of the filesystem
|
||||||
func (memoryFs) Hashes() hash.Set { return hash.Supported }
|
func (memoryFs) Hashes() hash.Set { return hash.Supported() }
|
||||||
|
|
||||||
// Features returns the optional features of this Fs
|
// Features returns the optional features of this Fs
|
||||||
func (memoryFs) Features() *fs.Features { return &fs.Features{} }
|
func (memoryFs) Features() *fs.Features { return &fs.Features{} }
|
||||||
|
|
|
@ -53,7 +53,7 @@ func TestMemoryFs(t *testing.T) {
|
||||||
assert.Equal(t, "", f.Root())
|
assert.Equal(t, "", f.Root())
|
||||||
assert.Equal(t, "memory", f.String())
|
assert.Equal(t, "memory", f.String())
|
||||||
assert.Equal(t, time.Nanosecond, f.Precision())
|
assert.Equal(t, time.Nanosecond, f.Precision())
|
||||||
assert.Equal(t, hash.Supported, f.Hashes())
|
assert.Equal(t, hash.Supported(), f.Hashes())
|
||||||
assert.Equal(t, &fs.Features{}, f.Features())
|
assert.Equal(t, &fs.Features{}, f.Features())
|
||||||
|
|
||||||
entries, err := f.List(context.Background(), "")
|
entries, err := f.List(context.Background(), "")
|
||||||
|
|
|
@ -980,15 +980,6 @@ func Sha1sum(ctx context.Context, f fs.Fs, w io.Writer) error {
|
||||||
return HashLister(ctx, hash.SHA1, f, w)
|
return HashLister(ctx, hash.SHA1, f, w)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DropboxHashSum list the Fs to the supplied writer
|
|
||||||
//
|
|
||||||
// Obeys includes and excludes
|
|
||||||
//
|
|
||||||
// Lists in parallel which may get them out of order
|
|
||||||
func DropboxHashSum(ctx context.Context, f fs.Fs, w io.Writer) error {
|
|
||||||
return HashLister(ctx, hash.Dropbox, f, w)
|
|
||||||
}
|
|
||||||
|
|
||||||
// hashSum returns the human readable hash for ht passed in. This may
|
// hashSum returns the human readable hash for ht passed in. This may
|
||||||
// be UNSUPPORTED or ERROR.
|
// be UNSUPPORTED or ERROR.
|
||||||
func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string {
|
func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string {
|
||||||
|
@ -1011,7 +1002,7 @@ func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string {
|
||||||
func HashLister(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error {
|
func HashLister(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error {
|
||||||
return ListFn(ctx, f, func(o fs.Object) {
|
return ListFn(ctx, f, func(o fs.Object) {
|
||||||
sum := hashSum(ctx, ht, o)
|
sum := hashSum(ctx, ht, o)
|
||||||
syncFprintf(w, "%*s %s\n", hash.Width[ht], sum, o.Remote())
|
syncFprintf(w, "%*s %s\n", hash.Width(ht), sum, o.Remote())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -225,23 +225,6 @@ func TestHashSums(t *testing.T) {
|
||||||
!strings.Contains(res, " potato2\n") {
|
!strings.Contains(res, " potato2\n") {
|
||||||
t.Errorf("potato2 missing: %q", res)
|
t.Errorf("potato2 missing: %q", res)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dropbox Hash Sum
|
|
||||||
|
|
||||||
buf.Reset()
|
|
||||||
err = operations.DropboxHashSum(context.Background(), r.Fremote, &buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
res = buf.String()
|
|
||||||
if !strings.Contains(res, "fc62b10ec59efa8041f5a6c924d7c91572c1bbda280d9e01312b660804df1d47 empty space\n") &&
|
|
||||||
!strings.Contains(res, " UNSUPPORTED empty space\n") &&
|
|
||||||
!strings.Contains(res, " empty space\n") {
|
|
||||||
t.Errorf("empty space missing: %q", res)
|
|
||||||
}
|
|
||||||
if !strings.Contains(res, "a979481df794fed9c3990a6a422e0b1044ac802c15fab13af9c687f8bdbee01a potato2\n") &&
|
|
||||||
!strings.Contains(res, " UNSUPPORTED potato2\n") &&
|
|
||||||
!strings.Contains(res, " potato2\n") {
|
|
||||||
t.Errorf("potato2 missing: %q", res)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSuffixName(t *testing.T) {
|
func TestSuffixName(t *testing.T) {
|
||||||
|
@ -1260,7 +1243,6 @@ func TestListFormat(t *testing.T) {
|
||||||
}{
|
}{
|
||||||
{hash.MD5, "0cc175b9c0f1b6a831c399e269772661"},
|
{hash.MD5, "0cc175b9c0f1b6a831c399e269772661"},
|
||||||
{hash.SHA1, "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"},
|
{hash.SHA1, "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"},
|
||||||
{hash.Dropbox, "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8"},
|
|
||||||
} {
|
} {
|
||||||
list.SetOutput(nil)
|
list.SetOutput(nil)
|
||||||
list.AddHash(test.ht)
|
list.AddHash(test.ht)
|
||||||
|
|
Loading…
Reference in a new issue