From 5b6f637461d4a91dc4aed79d5a305df92d8c47d5 Mon Sep 17 00:00:00 2001 From: Ivan Andreev Date: Fri, 21 May 2021 17:32:33 +0300 Subject: [PATCH] fs/hash: align hashsum names and update documentation (#5339) - Unify all hash names as lowercase alphanumerics without punctuation. - Legacy names continue to work but disappear from docs, they can be depreciated or dropped later. - Make rclone hashsum print supported hash list in case of wrong spelling. - Update documentation. Fixes #5071 Fixes #4841 --- backend/dropbox/dropbox.go | 2 +- backend/mailru/mailru.go | 2 +- backend/onedrive/onedrive.go | 2 +- cmd/hashsum/hashsum.go | 27 ++++--- docs/content/commands/rclone_hashsum.md | 13 ++- docs/content/flags.md | 0 fs/hash/hash.go | 101 +++++++++++------------- fs/hash/hash_test.go | 47 +++++++++-- fs/operations/operations_test.go | 8 +- fs/options_test.go | 2 +- 10 files changed, 122 insertions(+), 82 deletions(-) mode change 100755 => 100644 docs/content/flags.md diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 4a996806e..c007d0b33 100755 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -138,7 +138,7 @@ func getOauthConfig(m configmap.Mapper) *oauth2.Config { // Register with Fs func init() { - DbHashType = hash.RegisterHash("DropboxHash", 64, dbhash.New) + DbHashType = hash.RegisterHash("dropbox", "DropboxHash", 64, dbhash.New) fs.Register(&fs.RegInfo{ Name: "dropbox", Description: "Dropbox", diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index 0398dea32..8a437cf37 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -80,7 +80,7 @@ var oauthConfig = &oauth2.Config{ // Register with Fs func init() { - MrHashType = hash.RegisterHash("MailruHash", 40, mrhash.New) + MrHashType = hash.RegisterHash("mailru", "MailruHash", 40, mrhash.New) fs.Register(&fs.RegInfo{ Name: "mailru", Description: "Mail.ru Cloud", diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 16c1225fe..611960a54 100755 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -93,7 +93,7 @@ var ( // Register with Fs func init() { - QuickXorHashType = hash.RegisterHash("QuickXorHash", 40, quickxorhash.New) + QuickXorHashType = hash.RegisterHash("quickxor", "QuickXorHash", 40, quickxorhash.New) fs.Register(&fs.RegInfo{ Name: "onedrive", Description: "Microsoft OneDrive", diff --git a/cmd/hashsum/hashsum.go b/cmd/hashsum/hashsum.go index ce397611c..a69f8633b 100644 --- a/cmd/hashsum/hashsum.go +++ b/cmd/hashsum/hashsum.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "strings" "github.com/pkg/errors" "github.com/rclone/rclone/cmd" @@ -69,23 +70,17 @@ hashed locally enabling any hash for any remote. Run without a hash to see the list of all supported hashes, e.g. $ rclone hashsum - Supported hashes are: - * MD5 - * SHA-1 - * DropboxHash - * QuickXorHash - +` + hashListHelp(" ") + ` Then $ rclone hashsum MD5 remote:path + +Note that hash names are case insensitive. `, RunE: func(command *cobra.Command, args []string) error { cmd.CheckArgs(0, 2, command, args) if len(args) == 0 { - fmt.Printf("Supported hashes are:\n") - for _, ht := range hash.Supported().Array() { - fmt.Printf(" * %v\n", ht) - } + fmt.Print(hashListHelp("")) return nil } else if len(args) == 1 { return errors.New("need hash type and remote") @@ -93,6 +88,7 @@ Then var ht hash.Type err := ht.Set(args[0]) if err != nil { + fmt.Println(hashListHelp("")) return err } fsrc := cmd.NewFsSrc(args[1:]) @@ -111,3 +107,14 @@ Then return nil }, } + +func hashListHelp(indent string) string { + var help strings.Builder + help.WriteString(indent) + help.WriteString("Supported hashes are:\n") + for _, ht := range hash.Supported().Array() { + help.WriteString(indent) + fmt.Fprintf(&help, " * %v\n", ht.String()) + } + return help.String() +} diff --git a/docs/content/commands/rclone_hashsum.md b/docs/content/commands/rclone_hashsum.md index 2f150c69a..45cddc2e4 100644 --- a/docs/content/commands/rclone_hashsum.md +++ b/docs/content/commands/rclone_hashsum.md @@ -25,15 +25,20 @@ Run without a hash to see the list of all supported hashes, e.g. $ rclone hashsum Supported hashes are: - * MD5 - * SHA-1 - * DropboxHash - * QuickXorHash + * md5 + * sha1 + * whirlpool + * crc32 + * dropbox + * mailru + * quickxor Then $ rclone hashsum MD5 remote:path +Note that hash names are case insensitive. + ``` rclone hashsum remote:path [flags] diff --git a/docs/content/flags.md b/docs/content/flags.md old mode 100755 new mode 100644 diff --git a/fs/hash/hash.go b/fs/hash/hash.go index 8b49c2476..e0d791d95 100644 --- a/fs/hash/hash.go +++ b/fs/hash/hash.go @@ -20,25 +20,36 @@ type Type int type hashDefinition struct { width int name string + alias string newFunc func() hash.Hash hashType Type } -var hashes []*hashDefinition -var highestType Type = 1 +var ( + type2hash = map[Type]*hashDefinition{} + name2hash = map[string]*hashDefinition{} + alias2hash = map[string]*hashDefinition{} + supported = []Type{} +) // RegisterHash adds a new Hash to the list and returns it Type -func RegisterHash(name string, width int, newFunc func() hash.Hash) Type { +func RegisterHash(name, alias string, width int, newFunc func() hash.Hash) Type { + hashType := Type(1 << len(supported)) + supported = append(supported, hashType) + definition := &hashDefinition{ name: name, + alias: alias, width: width, newFunc: newFunc, - hashType: highestType, + hashType: hashType, } - hashes = append(hashes, definition) - highestType = highestType << 1 - return definition.hashType + type2hash[hashType] = definition + name2hash[name] = definition + alias2hash[alias] = definition + + return hashType } // ErrUnsupported should be returned by filesystem, @@ -63,31 +74,23 @@ var ( ) func init() { - MD5 = RegisterHash("MD5", 32, md5.New) - SHA1 = RegisterHash("SHA-1", 40, sha1.New) - Whirlpool = RegisterHash("Whirlpool", 128, whirlpool.New) - CRC32 = RegisterHash("CRC-32", 8, func() hash.Hash { return crc32.NewIEEE() }) + MD5 = RegisterHash("md5", "MD5", 32, md5.New) + SHA1 = RegisterHash("sha1", "SHA-1", 40, sha1.New) + Whirlpool = RegisterHash("whirlpool", "Whirlpool", 128, whirlpool.New) + CRC32 = RegisterHash("crc32", "CRC-32", 8, func() hash.Hash { return crc32.NewIEEE() }) } // Supported returns a set of all the supported hashes by // HashStream and MultiHasher. func Supported() Set { - var types []Type - for _, v := range hashes { - types = append(types, v.hashType) - } - - return NewHashSet(types...) + return NewHashSet(supported...) } // Width returns the width in characters for any HashType func Width(hashType Type) int { - for _, v := range hashes { - if v.hashType == hashType { - return v.width - } + if hash := type2hash[hashType]; hash != nil { + return hash.width } - return 0 } @@ -118,32 +121,29 @@ func StreamTypes(r io.Reader, set Set) (map[Type]string, error) { // The function will panic if the hash type is unknown. func (h Type) String() string { if h == None { - return "None" + return "none" } - - for _, v := range hashes { - if v.hashType == h { - return v.name - } + if hash := type2hash[h]; hash != nil { + return hash.name } - - err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h)) - panic(err) + panic(fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h))) } -// Set a Type from a flag +// Set a Type from a flag. +// Both name and alias are accepted. func (h *Type) Set(s string) error { - if s == "None" { + if s == "none" || s == "None" { *h = None + return nil } - - for _, v := range hashes { - if v.name == s { - *h = v.hashType - return nil - } + if hash := name2hash[strings.ToLower(s)]; hash != nil { + *h = hash.hashType + return nil + } + if hash := alias2hash[s]; hash != nil { + *h = hash.hashType + return nil } - return errors.Errorf("Unknown hash type %q", s) } @@ -159,23 +159,14 @@ func fromTypes(set Set) (map[Type]hash.Hash, error) { if !set.SubsetOf(Supported()) { return nil, errors.Errorf("requested set %08x contains unknown hash types", int(set)) } - var hashers = make(map[Type]hash.Hash) + hashers := map[Type]hash.Hash{} - types := set.Array() - for _, t := range types { - for _, v := range hashes { - if t != v.hashType { - continue - } - - hashers[t] = v.newFunc() - break - } - - if hashers[t] == nil { - err := fmt.Sprintf("internal error: Unsupported hash type %v", t) - panic(err) + for _, t := range set.Array() { + hash := type2hash[t] + if hash == nil { + panic(fmt.Sprintf("internal error: Unsupported hash type %v", t)) } + hashers[t] = hash.newFunc() } return hashers, nil diff --git a/fs/hash/hash_test.go b/fs/hash/hash_test.go index e7653dbb1..722e5d898 100644 --- a/fs/hash/hash_test.go +++ b/fs/hash/hash_test.go @@ -156,16 +156,53 @@ func TestHashStreamTypes(t *testing.T) { func TestHashSetStringer(t *testing.T) { h := hash.NewHashSet(hash.SHA1, hash.MD5) - assert.Equal(t, h.String(), "[MD5, SHA-1]") + assert.Equal(t, "[md5, sha1]", h.String()) h = hash.NewHashSet(hash.SHA1) - assert.Equal(t, h.String(), "[SHA-1]") + assert.Equal(t, "[sha1]", h.String()) h = hash.NewHashSet() - assert.Equal(t, h.String(), "[]") + assert.Equal(t, "[]", h.String()) } func TestHashStringer(t *testing.T) { h := hash.MD5 - assert.Equal(t, h.String(), "MD5") + assert.Equal(t, "md5", h.String()) + h = hash.SHA1 + assert.Equal(t, "sha1", h.String()) h = hash.None - assert.Equal(t, h.String(), "None") + assert.Equal(t, "none", h.String()) +} + +func TestHashSetter(t *testing.T) { + var ht hash.Type + + assert.NoError(t, ht.Set("none")) + assert.Equal(t, hash.None, ht) + assert.NoError(t, ht.Set("None")) + assert.Equal(t, hash.None, ht) + + assert.NoError(t, ht.Set("md5")) + assert.Equal(t, hash.MD5, ht) + assert.NoError(t, ht.Set("MD5")) + assert.Equal(t, hash.MD5, ht) + + assert.NoError(t, ht.Set("sha1")) + assert.Equal(t, hash.SHA1, ht) + assert.NoError(t, ht.Set("SHA-1")) + assert.Equal(t, hash.SHA1, ht) + + assert.NoError(t, ht.Set("SHA1")) + assert.Equal(t, hash.SHA1, ht) + assert.NoError(t, ht.Set("Sha1")) + assert.Equal(t, hash.SHA1, ht) + assert.Error(t, ht.Set("Sha-1")) +} + +func TestHashTypeStability(t *testing.T) { + assert.Equal(t, hash.Type(0), hash.None) + assert.Equal(t, hash.Type(1), hash.MD5) + assert.Equal(t, hash.Type(2), hash.SHA1) + + assert.True(t, hash.Supported().Contains(hash.MD5)) + assert.True(t, hash.Supported().Contains(hash.SHA1)) + assert.False(t, hash.Supported().Contains(hash.None)) } diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 603314b7d..f01b07a3e 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1206,10 +1206,10 @@ func TestListFormat(t *testing.T) { Format: "2006-01-02T15:04:05.000000000Z07:00"}, IsDir: false, Hashes: map[string]string{ - "MD5": "0cc175b9c0f1b6a831c399e269772661", - "SHA-1": "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", - "DropboxHash": "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8", - "QuickXorHash": "6100000000000000000000000100000000000000"}, + "md5": "0cc175b9c0f1b6a831c399e269772661", + "sha1": "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", + "dropbox": "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8", + "quickxor": "6100000000000000000000000100000000000000"}, ID: "fileID", OrigID: "fileOrigID", } diff --git a/fs/options_test.go b/fs/options_test.go index d02531078..52fc42ddb 100644 --- a/fs/options_test.go +++ b/fs/options_test.go @@ -115,7 +115,7 @@ func TestHTTPOption(t *testing.T) { func TestHashesOption(t *testing.T) { opt := &HashesOption{hash.Set(hash.MD5 | hash.SHA1)} var _ OpenOption = opt // check interface - assert.Equal(t, `HashesOption([MD5, SHA-1])`, opt.String()) + assert.Equal(t, `HashesOption([md5, sha1])`, opt.String()) key, value := opt.Header() assert.Equal(t, "", key) assert.Equal(t, "", value)