From e86f62c3e8cb6c6629aa3327c8aaf6828cc8db66 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 3 Jul 2017 15:05:27 +0100 Subject: [PATCH] Add rclone info internal command for testing out limits of remotes --- cmd/all/all.go | 1 + cmd/info/all.sh | 18 ++++ cmd/info/info.go | 214 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 233 insertions(+) create mode 100755 cmd/info/all.sh create mode 100644 cmd/info/info.go diff --git a/cmd/all/all.go b/cmd/all/all.go index bc9d546ab..7413eee45 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -18,6 +18,7 @@ import ( _ "github.com/ncw/rclone/cmd/delete" _ "github.com/ncw/rclone/cmd/genautocomplete" _ "github.com/ncw/rclone/cmd/gendocs" + _ "github.com/ncw/rclone/cmd/info" _ "github.com/ncw/rclone/cmd/listremotes" _ "github.com/ncw/rclone/cmd/ls" _ "github.com/ncw/rclone/cmd/ls2" diff --git a/cmd/info/all.sh b/cmd/info/all.sh new file mode 100755 index 000000000..b90354dbe --- /dev/null +++ b/cmd/info/all.sh @@ -0,0 +1,18 @@ +#!/bin/bash +exec rclone --check-normalization=true --check-control=true --check-length=true info \ + /tmp/testInfo \ + TestAmazonCloudDrive:testInfo \ + TestB2:testInfo \ + TestCryptDrive:testInfo \ + TestCryptSwift:testInfo \ + TestDrive:testInfo \ + TestDropbox:testInfo \ + TestGoogleCloudStorage:rclone-testinfo \ + TestOneDrive:testInfo \ + TestS3:rclone-testinfo \ + TestSftp:testInfo \ + TestSwift:testInfo \ + TestYandex:testInfo \ + TestFTP:testInfo + +# TestHubic:testInfo \ diff --git a/cmd/info/info.go b/cmd/info/info.go new file mode 100644 index 000000000..e4fe7be35 --- /dev/null +++ b/cmd/info/info.go @@ -0,0 +1,214 @@ +package info + +// FIXME once translations are implemented will need a no-escape +// option for Put so we can make these tests work agaig + +import ( + "bytes" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var ( + checkNormalization bool + checkControl bool + checkLength bool +) + +func init() { + cmd.Root.AddCommand(commandDefintion) + commandDefintion.Flags().BoolVarP(&checkNormalization, "check-normalization", "", true, "Check UTF-8 Normalization.") + commandDefintion.Flags().BoolVarP(&checkControl, "check-control", "", true, "Check control characters.") + commandDefintion.Flags().BoolVarP(&checkLength, "check-length", "", true, "Check max filename length.") +} + +var commandDefintion = &cobra.Command{ + Use: "info [remote:path]+", + Short: `Discovers file name limitations for paths.`, + Long: `rclone info discovers what filenames are possible to write to the +paths passed in and how long they can be. It can take some time. It +will write test files into the remote:path passed in. It outputs a bit +of go code for each one. +`, + Hidden: true, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(1, 1E6, command, args) + for i := range args { + f := cmd.NewFsDst(args[i : i+1]) + cmd.Run(false, false, command, func() error { + return readInfo(f) + }) + } + }, +} + +type results struct { + f fs.Fs + mu sync.Mutex + charNeedsEscaping map[rune]bool + maxFileLength int + canWriteUnnormalized bool + canReadUnnormalized bool + canReadRenormalized bool +} + +func newResults(f fs.Fs) *results { + return &results{ + f: f, + charNeedsEscaping: make(map[rune]bool), + } +} + +// Print the results to stdout +func (r *results) Print() { + fmt.Printf("// %s\n", r.f.Name()) + if checkControl { + escape := []string{} + for c, needsEscape := range r.charNeedsEscaping { + if needsEscape { + escape = append(escape, fmt.Sprintf("0x%02X", c)) + } + } + sort.Strings(escape) + fmt.Printf("charNeedsEscaping = []byte{\n") + fmt.Printf("\t%s\n", strings.Join(escape, ", ")) + fmt.Printf("}\n") + } + if checkLength { + fmt.Printf("maxFileLength = %d\n", r.maxFileLength) + } + if checkNormalization { + fmt.Printf("canWriteUnnormalized = %v\n", r.canWriteUnnormalized) + fmt.Printf("canReadUnnormalized = %v\n", r.canReadUnnormalized) + fmt.Printf("canReadRenormalized = %v\n", r.canReadRenormalized) + } +} + +// writeFile writes a file with some random contents +func (r *results) writeFile(path string) (fs.Object, error) { + contents := fstest.RandomString(50) + src := fs.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f) + return r.f.Put(bytes.NewBufferString(contents), src) +} + +// check whether normalization is enforced and check whether it is +// done on the files anyway +func (r *results) checkUTF8Normalization() { + unnormalized := "Héroique" + normalized := "Héroique" + _, err := r.writeFile(unnormalized) + if err != nil { + r.canWriteUnnormalized = false + return + } + r.canWriteUnnormalized = true + _, err = r.f.NewObject(unnormalized) + if err == nil { + r.canReadUnnormalized = true + } + _, err = r.f.NewObject(normalized) + if err == nil { + r.canReadRenormalized = true + } +} + +// check we can write file with the rune passed in +func (r *results) checkChar(c rune) { + fs.Infof(r.f, "Writing file 0x%02X", c) + path := fmt.Sprintf("0x%02X-%c-", c, c) + _, err := r.writeFile(path) + escape := false + if err != nil { + fs.Infof(r.f, "Couldn't write file 0x%02X", c) + } else { + fs.Infof(r.f, "OK writing file 0x%02X", c) + } + r.mu.Lock() + r.charNeedsEscaping[c] = escape + r.mu.Unlock() +} + +// check we can write a file with the control chars +func (r *results) checkControls() { + fs.Infof(r.f, "Trying to create control character file names") + // Concurrency control + tokens := make(chan struct{}, fs.Config.Checkers) + for i := 0; i < fs.Config.Checkers; i++ { + tokens <- struct{}{} + } + var wg sync.WaitGroup + for i := rune(0); i < 128; i++ { + if i == 0 || i == '/' { + // We're not even going to check NULL or / + r.charNeedsEscaping[i] = true + continue + } + wg.Add(1) + c := i + go func() { + defer wg.Done() + token := <-tokens + r.checkChar(c) + tokens <- token + }() + } + wg.Wait() + fs.Infof(r.f, "Done trying to create control character file names") +} + +// find the max file name size we can use +func (r *results) findMaxLength() { + const maxLen = 16 * 1024 + name := make([]byte, maxLen) + for i := range name { + name[i] = 'a' + } + // Find the first size of filename we can't write + i := sort.Search(len(name), func(i int) (fail bool) { + defer func() { + if err := recover(); err != nil { + fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err) + fail = true + } + }() + + path := string(name[:i]) + _, err := r.writeFile(path) + if err != nil { + fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err) + return true + } + fs.Infof(r.f, "Wrote file with name length %d", i) + return false + }) + r.maxFileLength = i - 1 + fs.Infof(r.f, "Max file length is %d", r.maxFileLength) +} + +func readInfo(f fs.Fs) error { + err := f.Mkdir("") + if err != nil { + return errors.Wrap(err, "couldn't mkdir") + } + r := newResults(f) + if checkControl { + r.checkControls() + } + if checkLength { + r.findMaxLength() + } + if checkNormalization { + r.checkUTF8Normalization() + } + r.Print() + return nil +}