Make rclone rmdirs command to delete empty directories - fixes #831

cmd/all/all.go

@@ -25,6 +25,7 @@ import (
 	_ "github.com/ncw/rclone/cmd/move"
 	_ "github.com/ncw/rclone/cmd/purge"
 	_ "github.com/ncw/rclone/cmd/rmdir"
+	_ "github.com/ncw/rclone/cmd/rmdirs"
 	_ "github.com/ncw/rclone/cmd/sha1sum"
 	_ "github.com/ncw/rclone/cmd/size"
 	_ "github.com/ncw/rclone/cmd/sync"

fs/operations.go

@@ -724,7 +724,11 @@ func Mkdir(f Fs, dir string) error {
 // count errors but may return one.
 func TryRmdir(f Fs, dir string) error {
 	if Config.DryRun {
-		Log(f, "Not deleting as dry run is set")
+		if dir != "" {
+			Log(dir, "Not deleting as dry run is set")
+		} else {
+			Log(f, "Not deleting as dry run is set")
+		}
 		return nil
 	}
 	return f.Rmdir(dir)
@@ -1068,3 +1072,62 @@ func Cat(f Fs, w io.Writer) error {
 		}
 	})
 }
+
+// Rmdirs removes any empty directories (or directories only
+// containing empty directories) under f, including f.
+func Rmdirs(f Fs) error {
+	list := NewLister().Start(f, "")
+	dirEmpty := make(map[string]bool)
+	dirEmpty[""] = true
+	for {
+		o, dir, err := list.Get()
+		if err != nil {
+			Stats.Error()
+			ErrorLog(f, "Failed to list: %v", err)
+			return err
+		} else if dir != nil {
+			// add a new directory as empty
+			dir := dir.Name
+			_, found := dirEmpty[dir]
+			if !found {
+				dirEmpty[dir] = true
+			}
+		} else if o != nil {
+			// mark the parents of the file as being non-empty
+			dir := o.Remote()
+			for dir != "" {
+				dir = path.Dir(dir)
+				if dir == "." || dir == "/" {
+					dir = ""
+				}
+				empty, found := dirEmpty[dir]
+				// End if we reach a directory which is non-empty
+				if found && !empty {
+					break
+				}
+				dirEmpty[dir] = false
+			}
+		} else {
+			// finished as dir == nil && o == nil
+			break
+		}
+	}
+	// Now delete the empty directories, starting from the longest path
+	var toDelete []string
+	for dir, empty := range dirEmpty {
+		if empty {
+			toDelete = append(toDelete, dir)
+		}
+	}
+	sort.Strings(toDelete)
+	for i := len(toDelete) - 1; i >= 0; i-- {
+		dir := toDelete[i]
+		err := TryRmdir(f, dir)
+		if err != nil {
+			Stats.Error()
+			ErrorLog(dir, "Failed to rmdir: %v", err)
+			return err
+		}
+	}
+	return nil
+}

fs/operations_test.go

@@ -638,3 +638,57 @@ func TestCat(t *testing.T) {
 		t.Errorf("Incorrect output from Cat: %q", res)
 	}
 }
+
+func TestRmdirs(t *testing.T) {
+	r := NewRun(t)
+	defer r.Finalise()
+	// Make some files and dirs we expect to keep
+	file1 := r.WriteObject("A1/B1/C1/one", "aaa", t1)
+	file2 := r.WriteObject("A1/two", "bbb", t2)
+	//..and dirs we expect to delete
+	require.NoError(t, fs.Mkdir(r.fremote, "A2"))
+	require.NoError(t, fs.Mkdir(r.fremote, "A1/B2"))
+	require.NoError(t, fs.Mkdir(r.fremote, "A1/B2/C2"))
+	require.NoError(t, fs.Mkdir(r.fremote, "A1/B1/C3"))
+	require.NoError(t, fs.Mkdir(r.fremote, "A3"))
+	require.NoError(t, fs.Mkdir(r.fremote, "A3/B3"))
+	require.NoError(t, fs.Mkdir(r.fremote, "A3/B3/C4"))
+
+	fstest.CheckListingWithPrecision(
+		t,
+		r.fremote,
+		[]fstest.Item{
+			file1, file2,
+		},
+		[]string{
+			"A1",
+			"A1/B1",
+			"A1/B1/C1",
+			"A2",
+			"A1/B2",
+			"A1/B2/C2",
+			"A1/B1/C3",
+			"A3",
+			"A3/B3",
+			"A3/B3/C4",
+		},
+		fs.Config.ModifyWindow,
+	)
+
+	require.NoError(t, fs.Rmdirs(r.fremote))
+
+	fstest.CheckListingWithPrecision(
+		t,
+		r.fremote,
+		[]fstest.Item{
+			file1, file2,
+		},
+		[]string{
+			"A1",
+			"A1/B1",
+			"A1/B1/C1",
+		},
+		fs.Config.ModifyWindow,
+	)
+
+}

fstest/fstest.go

@@ -145,6 +145,10 @@ func (is *Items) Done(t *testing.T) {
 
 // CheckListingWithPrecision checks the fs to see if it has the
 // expected contents with the given precision.
+//
+// If expectedDirs is non nil then we check those too.  Note that no
+// directories returned is also OK as some remotes don't return
+// directories.
 func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, expectedDirs []string, precision time.Duration) {
 	is := NewItems(items)
 	oldErrors := fs.Stats.GetErrors()
@@ -158,7 +162,7 @@ func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, expectedDirs
 		if err != nil && err != fs.ErrorDirNotFound {
 			t.Fatalf("Error listing: %v", err)
 		}
-		if len(objs) == len(items) {
+		if len(objs) == len(items) && (expectedDirs == nil || len(dirs) == 0 || len(dirs) == len(expectedDirs)) {
 			// Put an extra sleep in if we did any retries just to make sure it really
 			// is consistent (here is looking at you Amazon Drive!)
 			if i != 1 {