2020-08-09 17:31:04 +08:00
// Package makefiles builds a directory structure with the required
// number of files in of the required size.
package makefiles
import (
"io"
"log"
"math/rand"
"os"
"path/filepath"
2021-04-10 20:42:17 +08:00
"time"
2020-08-09 17:31:04 +08:00
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/test"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/lib/random"
"github.com/spf13/cobra"
)
var (
// Flags
numberOfFiles = 1000
averageFilesPerDirectory = 10
maxDepth = 10
minFileSize = fs . SizeSuffix ( 0 )
maxFileSize = fs . SizeSuffix ( 100 )
minFileNameLength = 4
maxFileNameLength = 12
2021-04-10 20:42:17 +08:00
seed = int64 ( 1 )
2020-08-09 17:31:04 +08:00
// Globals
2021-04-10 20:42:17 +08:00
randSource * rand . Rand
2020-08-09 17:31:04 +08:00
directoriesToCreate int
totalDirectories int
fileNames = map [ string ] struct { } { } // keep a note of which file name we've used already
)
func init ( ) {
test . Command . AddCommand ( commandDefinition )
cmdFlags := commandDefinition . Flags ( )
flags . IntVarP ( cmdFlags , & numberOfFiles , "files" , "" , numberOfFiles , "Number of files to create" )
flags . IntVarP ( cmdFlags , & averageFilesPerDirectory , "files-per-directory" , "" , averageFilesPerDirectory , "Average number of files per directory" )
flags . IntVarP ( cmdFlags , & maxDepth , "max-depth" , "" , maxDepth , "Maximum depth of directory hierarchy" )
flags . FVarP ( cmdFlags , & minFileSize , "min-file-size" , "" , "Minimum size of file to create" )
flags . FVarP ( cmdFlags , & maxFileSize , "max-file-size" , "" , "Maximum size of files to create" )
flags . IntVarP ( cmdFlags , & minFileNameLength , "min-name-length" , "" , minFileNameLength , "Minimum size of file names" )
flags . IntVarP ( cmdFlags , & maxFileNameLength , "max-name-length" , "" , maxFileNameLength , "Maximum size of file names" )
2021-04-10 20:42:17 +08:00
flags . Int64VarP ( cmdFlags , & seed , "seed" , "" , seed , "Seed for the random number generator (0 for random)" )
2020-08-09 17:31:04 +08:00
}
var commandDefinition = & cobra . Command {
Use : "makefiles <dir>" ,
Short : ` Make a random file hierarchy in <dir> ` ,
Run : func ( command * cobra . Command , args [ ] string ) {
cmd . CheckArgs ( 1 , 1 , command , args )
2021-04-10 20:42:17 +08:00
if seed == 0 {
seed = time . Now ( ) . UnixNano ( )
2021-04-10 23:59:30 +08:00
fs . Logf ( nil , "Using random seed = %d" , seed )
2021-04-10 20:42:17 +08:00
}
randSource = rand . New ( rand . NewSource ( seed ) )
2020-08-09 17:31:04 +08:00
outputDirectory := args [ 0 ]
directoriesToCreate = numberOfFiles / averageFilesPerDirectory
averageSize := ( minFileSize + maxFileSize ) / 2
2021-04-10 23:59:30 +08:00
start := time . Now ( )
fs . Logf ( nil , "Creating %d files of average size %v in %d directories in %q." , numberOfFiles , averageSize , directoriesToCreate , outputDirectory )
2020-08-09 17:31:04 +08:00
root := & dir { name : outputDirectory , depth : 1 }
for totalDirectories < directoriesToCreate {
root . createDirectories ( )
}
dirs := root . list ( "" , [ ] string { } )
2021-04-10 23:59:30 +08:00
totalBytes := int64 ( 0 )
2020-08-09 17:31:04 +08:00
for i := 0 ; i < numberOfFiles ; i ++ {
2021-04-10 20:42:17 +08:00
dir := dirs [ randSource . Intn ( len ( dirs ) ) ]
2021-04-10 23:59:30 +08:00
totalBytes += writeFile ( dir , fileName ( ) )
2020-08-09 17:31:04 +08:00
}
2021-04-10 23:59:30 +08:00
dt := time . Since ( start )
fs . Logf ( nil , "Written %viB in %v at %viB/s." , fs . SizeSuffix ( totalBytes ) , dt . Round ( time . Millisecond ) , fs . SizeSuffix ( ( totalBytes * int64 ( time . Second ) ) / int64 ( dt ) ) )
2020-08-09 17:31:04 +08:00
} ,
}
// fileName creates a unique random file or directory name
func fileName ( ) ( name string ) {
for {
2021-04-10 20:42:17 +08:00
length := randSource . Intn ( maxFileNameLength - minFileNameLength ) + minFileNameLength
name = random . StringFn ( length , randSource . Intn )
2020-08-09 17:31:04 +08:00
if _ , found := fileNames [ name ] ; ! found {
break
}
}
fileNames [ name ] = struct { } { }
return name
}
// dir is a directory in the directory hierarchy being built up
type dir struct {
name string
depth int
children [ ] * dir
parent * dir
}
// Create a random directory hierarchy under d
func ( d * dir ) createDirectories ( ) {
for totalDirectories < directoriesToCreate {
newDir := & dir {
name : fileName ( ) ,
depth : d . depth + 1 ,
parent : d ,
}
d . children = append ( d . children , newDir )
totalDirectories ++
2021-04-10 20:42:17 +08:00
switch randSource . Intn ( 4 ) {
2020-08-09 17:31:04 +08:00
case 0 :
if d . depth < maxDepth {
newDir . createDirectories ( )
}
case 1 :
return
}
}
return
}
// list the directory hierarchy
func ( d * dir ) list ( path string , output [ ] string ) [ ] string {
dirPath := filepath . Join ( path , d . name )
output = append ( output , dirPath )
for _ , subDir := range d . children {
output = subDir . list ( dirPath , output )
}
return output
}
// writeFile writes a random file at dir/name
2021-04-10 23:59:30 +08:00
func writeFile ( dir , name string ) int64 {
2020-08-09 17:31:04 +08:00
err := os . MkdirAll ( dir , 0777 )
if err != nil {
log . Fatalf ( "Failed to make directory %q: %v" , dir , err )
}
path := filepath . Join ( dir , name )
fd , err := os . Create ( path )
if err != nil {
log . Fatalf ( "Failed to open file %q: %v" , path , err )
}
2021-04-10 20:42:17 +08:00
size := randSource . Int63n ( int64 ( maxFileSize - minFileSize ) ) + int64 ( minFileSize )
_ , err = io . CopyN ( fd , randSource , size )
2020-08-09 17:31:04 +08:00
if err != nil {
log . Fatalf ( "Failed to write %v bytes to file %q: %v" , size , path , err )
}
err = fd . Close ( )
if err != nil {
log . Fatalf ( "Failed to close file %q: %v" , path , err )
}
2021-04-10 23:59:30 +08:00
fs . Infof ( path , "Written file size %v" , fs . SizeSuffix ( size ) )
return size
2020-08-09 17:31:04 +08:00
}