package fs import ( "fmt" "io" "sync" "testing" "time" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) type ( listResult struct { entries DirEntries err error } listResults map[string]listResult errorMap map[string]error listDirs struct { mu sync.Mutex t *testing.T fs Fs includeAll bool results listResults walkResults listResults walkErrors errorMap finalError error checkMaps bool maxLevel int } ) var errNotImpl = errors.New("not implemented") type mockObject string func (o mockObject) String() string { return string(o) } func (o mockObject) Fs() Info { return nil } func (o mockObject) Remote() string { return string(o) } func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl } func (o mockObject) ModTime() (t time.Time) { return t } func (o mockObject) Size() int64 { return 0 } func (o mockObject) Storable() bool { return true } func (o mockObject) SetModTime(time.Time) error { return errNotImpl } func (o mockObject) Open(options ...OpenOption) (io.ReadCloser, error) { return nil, errNotImpl } func (o mockObject) Update(in io.Reader, src ObjectInfo, options ...OpenOption) error { return errNotImpl } func (o mockObject) Remove() error { return errNotImpl } type unknownDirEntry string func (o unknownDirEntry) String() string { return string(o) } func (o unknownDirEntry) Remote() string { return string(o) } func (o unknownDirEntry) ModTime() (t time.Time) { return t } func (o unknownDirEntry) Size() int64 { return 0 } func newListDirs(t *testing.T, f Fs, includeAll bool, results listResults, walkErrors errorMap, finalError error) *listDirs { return &listDirs{ t: t, fs: f, includeAll: includeAll, results: results, walkErrors: walkErrors, walkResults: listResults{}, finalError: finalError, checkMaps: true, maxLevel: -1, } } // NoCheckMaps marks the maps as to be ignored at the end func (ls *listDirs) NoCheckMaps() *listDirs { ls.checkMaps = false return ls } // SetLevel(1) turns off recursion func (ls *listDirs) SetLevel(maxLevel int) *listDirs { ls.maxLevel = maxLevel return ls } // ListDir returns the expected listing for the directory func (ls *listDirs) ListDir(f Fs, includeAll bool, dir string) (entries DirEntries, err error) { ls.mu.Lock() defer ls.mu.Unlock() assert.Equal(ls.t, ls.fs, f) assert.Equal(ls.t, ls.includeAll, includeAll) // Fetch results for this path result, ok := ls.results[dir] if !ok { ls.t.Errorf("Unexpected list of %q", dir) return nil, errors.New("unexpected list") } delete(ls.results, dir) // Put expected results for call of WalkFn ls.walkResults[dir] = result return result.entries, result.err } // ListR returns the expected listing for the directory using ListR func (ls *listDirs) ListR(dir string, callback ListRCallback) (err error) { ls.mu.Lock() defer ls.mu.Unlock() var errorReturn error for dirPath, result := range ls.results { // Put expected results for call of WalkFn // Note that we don't call the function at all if we got an error if result.err != nil { errorReturn = result.err } if errorReturn == nil { err = callback(result.entries) require.NoError(ls.t, err) ls.walkResults[dirPath] = result } } ls.results = listResults{} return errorReturn } // IsFinished checks everything expected was used up func (ls *listDirs) IsFinished() { if ls.checkMaps { assert.Equal(ls.t, errorMap{}, ls.walkErrors) assert.Equal(ls.t, listResults{}, ls.results) assert.Equal(ls.t, listResults{}, ls.walkResults) } } // WalkFn is called by the walk to test the expectations func (ls *listDirs) WalkFn(dir string, entries DirEntries, err error) error { ls.mu.Lock() defer ls.mu.Unlock() // ls.t.Logf("WalkFn(%q, %v, %q)", dir, entries, err) // Fetch expected entries and err result, ok := ls.walkResults[dir] if !ok { ls.t.Errorf("Unexpected walk of %q (result not found)", dir) return errors.New("result not found") } delete(ls.walkResults, dir) // Check arguments are as expected assert.Equal(ls.t, result.entries, entries) assert.Equal(ls.t, result.err, err) // Fetch return value returnErr, ok := ls.walkErrors[dir] if !ok { ls.t.Errorf("Unexpected walk of %q (error not found)", dir) return errors.New("error not found") } delete(ls.walkErrors, dir) return returnErr } // Walk does the walk and tests the expectations func (ls *listDirs) Walk() { err := walk(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListDir) assert.Equal(ls.t, ls.finalError, err) ls.IsFinished() } // WalkR does the walkR and tests the expectations func (ls *listDirs) WalkR() { err := walkR(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListR) assert.Equal(ls.t, ls.finalError, err) if ls.finalError == nil { ls.IsFinished() } } func newDir(name string) Directory { return NewDir(name, time.Time{}) } func testWalkEmpty(t *testing.T) *listDirs { return newListDirs(t, nil, false, listResults{ "": {entries: DirEntries{}, err: nil}, }, errorMap{ "": nil, }, nil, ) } func TestWalkEmpty(t *testing.T) { testWalkEmpty(t).Walk() } func TestWalkREmpty(t *testing.T) { testWalkEmpty(t).WalkR() } func testWalkEmptySkip(t *testing.T) *listDirs { return newListDirs(t, nil, true, listResults{ "": {entries: DirEntries{}, err: nil}, }, errorMap{ "": ErrorSkipDir, }, nil, ) } func TestWalkEmptySkip(t *testing.T) { testWalkEmptySkip(t).Walk() } func TestWalkREmptySkip(t *testing.T) { testWalkEmptySkip(t).WalkR() } func testWalkNotFound(t *testing.T) *listDirs { return newListDirs(t, nil, true, listResults{ "": {err: ErrorDirNotFound}, }, errorMap{ "": ErrorDirNotFound, }, ErrorDirNotFound, ) } func TestWalkNotFound(t *testing.T) { testWalkNotFound(t).Walk() } func TestWalkRNotFound(t *testing.T) { testWalkNotFound(t).WalkR() } func TestWalkNotFoundMaskError(t *testing.T) { // this doesn't work for WalkR newListDirs(t, nil, true, listResults{ "": {err: ErrorDirNotFound}, }, errorMap{ "": nil, }, nil, ).Walk() } func TestWalkNotFoundSkipkError(t *testing.T) { // this doesn't work for WalkR newListDirs(t, nil, true, listResults{ "": {err: ErrorDirNotFound}, }, errorMap{ "": ErrorSkipDir, }, nil, ).Walk() } func testWalkLevels(t *testing.T, maxLevel int) *listDirs { da := newDir("a") oA := mockObject("A") db := newDir("a/b") oB := mockObject("a/B") dc := newDir("a/b/c") oC := mockObject("a/b/C") dd := newDir("a/b/c/d") oD := mockObject("a/b/c/D") return newListDirs(t, nil, false, listResults{ "": {entries: DirEntries{oA, da}, err: nil}, "a": {entries: DirEntries{oB, db}, err: nil}, "a/b": {entries: DirEntries{oC, dc}, err: nil}, "a/b/c": {entries: DirEntries{oD, dd}, err: nil}, "a/b/c/d": {entries: DirEntries{}, err: nil}, }, errorMap{ "": nil, "a": nil, "a/b": nil, "a/b/c": nil, "a/b/c/d": nil, }, nil, ).SetLevel(maxLevel) } func TestWalkLevels(t *testing.T) { testWalkLevels(t, -1).Walk() } func TestWalkRLevels(t *testing.T) { testWalkLevels(t, -1).WalkR() } func TestWalkLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).Walk() } func TestWalkRLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).WalkR() } func TestWalkNDirTree(t *testing.T) { ls := testWalkLevels(t, -1) entries, err := walkNDirTree(nil, "", ls.includeAll, ls.maxLevel, ls.ListDir) require.NoError(t, err) assert.Equal(t, `/ A a/ a/ B b/ a/b/ C c/ a/b/c/ D d/ a/b/c/d/ `, entries.String()) } func testWalkLevelsNoRecursive(t *testing.T) *listDirs { da := newDir("a") oA := mockObject("A") return newListDirs(t, nil, false, listResults{ "": {entries: DirEntries{oA, da}, err: nil}, }, errorMap{ "": nil, }, nil, ).SetLevel(1) } func TestWalkLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).Walk() } func TestWalkRLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).WalkR() } func testWalkLevels2(t *testing.T) *listDirs { da := newDir("a") oA := mockObject("A") db := newDir("a/b") oB := mockObject("a/B") return newListDirs(t, nil, false, listResults{ "": {entries: DirEntries{oA, da}, err: nil}, "a": {entries: DirEntries{oB, db}, err: nil}, }, errorMap{ "": nil, "a": nil, }, nil, ).SetLevel(2) } func TestWalkLevels2(t *testing.T) { testWalkLevels2(t).Walk() } func TestWalkRLevels2(t *testing.T) { testWalkLevels2(t).WalkR() } func testWalkSkip(t *testing.T) *listDirs { da := newDir("a") db := newDir("a/b") dc := newDir("a/b/c") return newListDirs(t, nil, false, listResults{ "": {entries: DirEntries{da}, err: nil}, "a": {entries: DirEntries{db}, err: nil}, "a/b": {entries: DirEntries{dc}, err: nil}, }, errorMap{ "": nil, "a": nil, "a/b": ErrorSkipDir, }, nil, ) } func TestWalkSkip(t *testing.T) { testWalkSkip(t).Walk() } func TestWalkRSkip(t *testing.T) { testWalkSkip(t).WalkR() } func testWalkErrors(t *testing.T) *listDirs { lr := listResults{} em := errorMap{} de := make(DirEntries, 10) for i := range de { path := string('0' + i) de[i] = newDir(path) lr[path] = listResult{entries: nil, err: ErrorDirNotFound} em[path] = ErrorDirNotFound } lr[""] = listResult{entries: de, err: nil} em[""] = nil return newListDirs(t, nil, true, lr, em, ErrorDirNotFound, ).NoCheckMaps() } func TestWalkErrors(t *testing.T) { testWalkErrors(t).Walk() } func TestWalkRErrors(t *testing.T) { testWalkErrors(t).WalkR() } var errorBoom = errors.New("boom") func makeTree(level int, terminalErrors bool) (listResults, errorMap) { lr := listResults{} em := errorMap{} var fill func(path string, level int) fill = func(path string, level int) { de := DirEntries{} if level > 0 { for _, a := range "0123456789" { subPath := string(a) if path != "" { subPath = path + "/" + subPath } de = append(de, newDir(subPath)) fill(subPath, level-1) } } lr[path] = listResult{entries: de, err: nil} em[path] = nil if level == 0 && terminalErrors { em[path] = errorBoom } } fill("", level) return lr, em } func testWalkMulti(t *testing.T) *listDirs { lr, em := makeTree(3, false) return newListDirs(t, nil, true, lr, em, nil, ) } func TestWalkMulti(t *testing.T) { testWalkMulti(t).Walk() } func TestWalkRMulti(t *testing.T) { testWalkMulti(t).WalkR() } func testWalkMultiErrors(t *testing.T) *listDirs { lr, em := makeTree(3, true) return newListDirs(t, nil, true, lr, em, errorBoom, ).NoCheckMaps() } func TestWalkMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() } func TestWalkRMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() } // a very simple listRcallback function func makeListRCallback(entries DirEntries, err error) ListRFn { return func(dir string, callback ListRCallback) error { if err == nil { err = callback(entries) } return err } } func TestWalkRDirTree(t *testing.T) { for _, test := range []struct { entries DirEntries want string err error root string level int }{ {DirEntries{}, "/\n", nil, "", -1}, {DirEntries{mockObject("a")}, `/ a `, nil, "", -1}, {DirEntries{mockObject("a/b")}, `/ a/ a/ b `, nil, "", -1}, {DirEntries{mockObject("a/b/c/d")}, `/ a/ a/ b/ a/b/ c/ a/b/c/ d `, nil, "", -1}, {DirEntries{mockObject("a")}, "", errorBoom, "", -1}, {DirEntries{ mockObject("0/1/2/3"), mockObject("4/5/6/7"), mockObject("8/9/a/b"), mockObject("c/d/e/f"), mockObject("g/h/i/j"), mockObject("k/l/m/n"), mockObject("o/p/q/r"), mockObject("s/t/u/v"), mockObject("w/x/y/z"), }, `/ 0/ 4/ 8/ c/ g/ k/ o/ s/ w/ 0/ 1/ 0/1/ 2/ 0/1/2/ 3 4/ 5/ 4/5/ 6/ 4/5/6/ 7 8/ 9/ 8/9/ a/ 8/9/a/ b c/ d/ c/d/ e/ c/d/e/ f g/ h/ g/h/ i/ g/h/i/ j k/ l/ k/l/ m/ k/l/m/ n o/ p/ o/p/ q/ o/p/q/ r s/ t/ s/t/ u/ s/t/u/ v w/ x/ w/x/ y/ w/x/y/ z `, nil, "", -1}, {DirEntries{ mockObject("a/b/c/d/e/f1"), mockObject("a/b/c/d/e/f2"), mockObject("a/b/c/d/e/f3"), }, `a/b/c/ d/ a/b/c/d/ e/ a/b/c/d/e/ f1 f2 f3 `, nil, "a/b/c", -1}, {DirEntries{ mockObject("A"), mockObject("a/B"), mockObject("a/b/C"), mockObject("a/b/c/D"), mockObject("a/b/c/d/E"), }, `/ A a/ a/ B b/ `, nil, "", 2}, {DirEntries{ mockObject("a/b/c"), mockObject("a/b/c/d/e"), }, `/ a/ a/ b/ `, nil, "", 2}, } { r, err := walkRDirTree(nil, test.root, true, test.level, makeListRCallback(test.entries, test.err)) assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test)) assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test)) } } func TestWalkRDirTreeExclude(t *testing.T) { for _, test := range []struct { entries DirEntries want string err error root string level int excludeFile string includeAll bool }{ {DirEntries{mockObject("a"), mockObject("ignore")}, "", nil, "", -1, "ignore", false}, {DirEntries{mockObject("a")}, `/ a `, nil, "", -1, "ignore", false}, {DirEntries{ mockObject("a"), mockObject("b/b"), mockObject("b/.ignore"), }, `/ a `, nil, "", -1, ".ignore", false}, {DirEntries{ mockObject("a"), mockObject("b/.ignore"), mockObject("b/b"), }, `/ a b/ b/ .ignore b `, nil, "", -1, ".ignore", true}, {DirEntries{ mockObject("a"), mockObject("b/b"), mockObject("b/c/d/e"), mockObject("b/c/ign"), mockObject("b/c/x"), }, `/ a b/ b/ b `, nil, "", -1, "ign", false}, {DirEntries{ mockObject("a"), mockObject("b/b"), mockObject("b/c/d/e"), mockObject("b/c/ign"), mockObject("b/c/x"), }, `/ a b/ b/ b c/ b/c/ d/ ign x b/c/d/ e `, nil, "", -1, "ign", true}, } { Config.Filter.ExcludeFile = test.excludeFile r, err := walkRDirTree(nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err)) assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test)) assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test)) } // Set to default value, to avoid side effects Config.Filter.ExcludeFile = "" }