rclone/cmd/info/info.go

package info

// FIXME once translations are implemented will need a no-escape
// option for Put so we can make these tests work agaig

import (
	"bytes"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fstest"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var (
	checkNormalization bool
	checkControl       bool
	checkLength        bool
)

func init() {
	cmd.Root.AddCommand(commandDefintion)
	commandDefintion.Flags().BoolVarP(&checkNormalization, "check-normalization", "", true, "Check UTF-8 Normalization.")
	commandDefintion.Flags().BoolVarP(&checkControl, "check-control", "", true, "Check control characters.")
	commandDefintion.Flags().BoolVarP(&checkLength, "check-length", "", true, "Check max filename length.")
}

var commandDefintion = &cobra.Command{
	Use:   "info [remote:path]+",
	Short: `Discovers file name limitations for paths.`,
	Long: `rclone info discovers what filenames are possible to write to the
paths passed in and how long they can be.  It can take some time.  It
will write test files into the remote:path passed in.  It outputs a bit
of go code for each one.
`,
	Hidden: true,
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(1, 1E6, command, args)
		for i := range args {
			f := cmd.NewFsDst(args[i : i+1])
			cmd.Run(false, false, command, func() error {
				return readInfo(f)
			})
		}
	},
}

type results struct {
	f                    fs.Fs
	mu                   sync.Mutex
	charNeedsEscaping    map[rune]bool
	maxFileLength        int
	canWriteUnnormalized bool
	canReadUnnormalized  bool
	canReadRenormalized  bool
}

func newResults(f fs.Fs) *results {
	return &results{
		f:                 f,
		charNeedsEscaping: make(map[rune]bool),
	}
}

// Print the results to stdout
func (r *results) Print() {
	fmt.Printf("// %s\n", r.f.Name())
	if checkControl {
		escape := []string{}
		for c, needsEscape := range r.charNeedsEscaping {
			if needsEscape {
				escape = append(escape, fmt.Sprintf("0x%02X", c))
			}
		}
		sort.Strings(escape)
		fmt.Printf("charNeedsEscaping = []byte{\n")
		fmt.Printf("\t%s\n", strings.Join(escape, ", "))
		fmt.Printf("}\n")
	}
	if checkLength {
		fmt.Printf("maxFileLength = %d\n", r.maxFileLength)
	}
	if checkNormalization {
		fmt.Printf("canWriteUnnormalized = %v\n", r.canWriteUnnormalized)
		fmt.Printf("canReadUnnormalized   = %v\n", r.canReadUnnormalized)
		fmt.Printf("canReadRenormalized   = %v\n", r.canReadRenormalized)
	}
}

// writeFile writes a file with some random contents
func (r *results) writeFile(path string) (fs.Object, error) {
	contents := fstest.RandomString(50)
	src := fs.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f)
	return r.f.Put(bytes.NewBufferString(contents), src)
}

// check whether normalization is enforced and check whether it is
// done on the files anyway
func (r *results) checkUTF8Normalization() {
	unnormalized := "Héroique"
	normalized := "Héroique"
	_, err := r.writeFile(unnormalized)
	if err != nil {
		r.canWriteUnnormalized = false
		return
	}
	r.canWriteUnnormalized = true
	_, err = r.f.NewObject(unnormalized)
	if err == nil {
		r.canReadUnnormalized = true
	}
	_, err = r.f.NewObject(normalized)
	if err == nil {
		r.canReadRenormalized = true
	}
}

// check we can write file with the rune passed in
func (r *results) checkChar(c rune) {
	fs.Infof(r.f, "Writing file 0x%02X", c)
	path := fmt.Sprintf("0x%02X-%c-", c, c)
	_, err := r.writeFile(path)
	escape := false
	if err != nil {
		fs.Infof(r.f, "Couldn't write file 0x%02X", c)
	} else {
		fs.Infof(r.f, "OK writing file 0x%02X", c)
	}
	r.mu.Lock()
	r.charNeedsEscaping[c] = escape
	r.mu.Unlock()
}

// check we can write a file with the control chars
func (r *results) checkControls() {
	fs.Infof(r.f, "Trying to create control character file names")
	// Concurrency control
	tokens := make(chan struct{}, fs.Config.Checkers)
	for i := 0; i < fs.Config.Checkers; i++ {
		tokens <- struct{}{}
	}
	var wg sync.WaitGroup
	for i := rune(0); i < 128; i++ {
		if i == 0 || i == '/' {
			// We're not even going to check NULL or /
			r.charNeedsEscaping[i] = true
			continue
		}
		wg.Add(1)
		c := i
		go func() {
			defer wg.Done()
			token := <-tokens
			r.checkChar(c)
			tokens <- token
		}()
	}
	wg.Wait()
	fs.Infof(r.f, "Done trying to create control character file names")
}

// find the max file name size we can use
func (r *results) findMaxLength() {
	const maxLen = 16 * 1024
	name := make([]byte, maxLen)
	for i := range name {
		name[i] = 'a'
	}
	// Find the first size of filename we can't write
	i := sort.Search(len(name), func(i int) (fail bool) {
		defer func() {
			if err := recover(); err != nil {
				fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
				fail = true
			}
		}()

		path := string(name[:i])
		_, err := r.writeFile(path)
		if err != nil {
			fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
			return true
		}
		fs.Infof(r.f, "Wrote file with name length %d", i)
		return false
	})
	r.maxFileLength = i - 1
	fs.Infof(r.f, "Max file length is %d", r.maxFileLength)
}

func readInfo(f fs.Fs) error {
	err := f.Mkdir("")
	if err != nil {
		return errors.Wrap(err, "couldn't mkdir")
	}
	r := newResults(f)
	if checkControl {
		r.checkControls()
	}
	if checkLength {
		r.findMaxLength()
	}
	if checkNormalization {
		r.checkUTF8Normalization()
	}
	r.Print()
	return nil
}