test: add "rclone test histogram" for file name distribution stats

cmd/all/all.go

@@ -53,6 +53,7 @@ import (
 	_ "github.com/rclone/rclone/cmd/size"
 	_ "github.com/rclone/rclone/cmd/sync"
 	_ "github.com/rclone/rclone/cmd/test"
+	_ "github.com/rclone/rclone/cmd/test/histogram"
 	_ "github.com/rclone/rclone/cmd/test/info"
 	_ "github.com/rclone/rclone/cmd/test/makefiles"
 	_ "github.com/rclone/rclone/cmd/test/memory"

cmd/test/histogram/histogram.go

@@ -0,0 +1,59 @@
+package histogram
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/cmd/test"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/walk"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	test.Command.AddCommand(commandDefinition)
+}
+
+var commandDefinition = &cobra.Command{
+	Use:   "histogram [remote:path]",
+	Short: `Makes a histogram of file name characters.`,
+	Long: `This command outputs JSON which shows the histogram of characters used
+in filenames in the remote:path specified.
+
+The data doesn't contain any identifying information but is useful for
+the rclone developers when developing filename compression.
+`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		f := cmd.NewFsDir(args)
+		ctx := context.Background()
+		ci := fs.GetConfig(ctx)
+		cmd.Run(false, false, command, func() error {
+			var hist [256]int64
+			err := walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
+				for _, entry := range entries {
+					base := path.Base(entry.Remote())
+					for i := range base {
+						hist[base[i]]++
+					}
+				}
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+			enc := json.NewEncoder(os.Stdout)
+			// enc.SetIndent("", "\t")
+			err = enc.Encode(&hist)
+			if err != nil {
+				return err
+			}
+			fmt.Println()
+			return nil
+		})
+	},
+}