From a1a780e8476554be31ffd8fad55391477fd8c820 Mon Sep 17 00:00:00 2001
From: Klaus Post <klauspost@gmail.com>
Date: Sat, 12 Sep 2015 21:59:14 +0200
Subject: [PATCH] Read folders in separate goroutines.

As proposed in the FIXME, read folders in parallel.
This appears to fix "Next token is expired" on very big directories.
The only downside is that this doesn't abort at once if an error is found.
I added some logging, so there is some output for "-v".
---
 amazonclouddrive/amazonclouddrive.go | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go
index f8454005c..378e227d9 100644
--- a/amazonclouddrive/amazonclouddrive.go
+++ b/amazonclouddrive/amazonclouddrive.go
@@ -28,6 +28,7 @@ import (
 	"github.com/ncw/rclone/oauthutil"
 	"github.com/ncw/rclone/pacer"
 	"golang.org/x/oauth2"
+	"sync"
 )
 
 const (
@@ -353,16 +354,23 @@ OUTER:
 func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
 	var subError error
 	// Make the API request
+	var wg sync.WaitGroup
 	_, err := f.listAll(dirId, "", false, false, func(node *acd.Node) bool {
 		// Recurse on directories
-		// FIXME should do this in parallel
-		// use a wg to sync then collect error
 		switch *node.Kind {
 		case folderKind:
-			subError = f.listDirRecursive(*node.Id, path+*node.Name+"/", out)
-			if subError != nil {
-				return true
-			}
+			wg.Add(1)
+			folder := path + *node.Name + "/"
+			fs.Debug(f, "Reading %s", folder)
+			go func() {
+				defer wg.Done()
+				err := f.listDirRecursive(*node.Id, folder, out)
+				if err != nil {
+					subError = err
+					fs.ErrorLog(f, "Error reading %s:%s", folder, err)
+				}
+			}()
+			return false
 		case fileKind:
 			if fs := f.newFsObjectWithInfo(path+*node.Name, node); fs != nil {
 				out <- fs
@@ -372,6 +380,8 @@ func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan)
 		}
 		return false
 	})
+	wg.Wait()
+	fs.Debug(f, "Finished reading %s", path)
 	if err != nil {
 		return err
 	}