Implement Amazon Cloud Drive - fixes #45

* Optional interfaces Copier, Mover, DirMover not done

README.md

@@ -15,6 +15,7 @@ Rclone is a command line program to sync files and directories to and from
   * Openstack Swift / Rackspace cloud files / Memset Memstore
   * Dropbox
   * Google Cloud Storage
+  * Amazon Cloud Drive
   * The local filesystem
 
 Features

amazonclouddrive/amazonclouddrive.go

@@ -0,0 +1,622 @@
+// Amazon Cloud Drive interface
+package amazonclouddrive
+
+/*
+
+FIXME make searching for directory in id and file in id more efficient
+- use the name: search parameter - remember the escaping rules
+- use Folder GetNode and GetFile
+
+FIXME make the default for no files and no dirs be (FILE & FOLDER) so
+we ignore assets completely!
+
+FIXME detect 429 errors and return error with fs.RetryErrorf?
+
+*/
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/ncw/go-acd"
+	"github.com/ncw/rclone/dircache"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/oauthutil"
+	"golang.org/x/oauth2"
+)
+
+const (
+	rcloneClientID     = "amzn1.application-oa2-client.6bf18d2d1f5b485c94c8988bb03ad0e7"
+	rcloneClientSecret = "k8/NyszKm5vEkZXAwsbGkd6C3NrbjIqMg4qEhIeF14Szub2wur+/teS3ubXgsLe9//+tr/qoqK+lq6mg8vWkoA=="
+	bindAddress        = "127.0.0.1:53682"
+	redirectURL        = "http://" + bindAddress + "/"
+	folderKind         = "FOLDER"
+	fileKind           = "FILE"
+	assetKind          = "ASSET"
+	statusAvailable    = "AVAILABLE"
+	timeFormat         = time.RFC3339 // 2014-03-07T22:31:12.173Z
+)
+
+// Globals
+var (
+	// Description of how to auth for this app
+	acdConfig = &oauth2.Config{
+		Scopes: []string{"clouddrive:read_all", "clouddrive:write"},
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://www.amazon.com/ap/oa",
+			TokenURL: "https://api.amazon.com/auth/o2/token",
+		},
+		ClientID:     rcloneClientID,
+		ClientSecret: fs.Reveal(rcloneClientSecret),
+		RedirectURL:  redirectURL,
+	}
+	FIXME = fmt.Errorf("FIXME not implemented")
+)
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.FsInfo{
+		Name:  "amazon cloud drive",
+		NewFs: NewFs,
+		Config: func(name string) {
+			err := oauthutil.ConfigWithWebserver(name, acdConfig, bindAddress)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+			}
+		},
+		Options: []fs.Option{{
+			Name: "client_id",
+			Help: "Amazon Application Client Id - leave blank to use rclone's.",
+		}, {
+			Name: "client_secret",
+			Help: "Amazon Application Client Secret - leave blank to use rclone's.",
+		}},
+	})
+}
+
+// FsAcd represents a remote acd server
+type FsAcd struct {
+	name     string             // name of this remote
+	c        *acd.Client        // the connection to the acd server
+	root     string             // the path we are working on
+	dirCache *dircache.DirCache // Map of directory path to directory id
+}
+
+// FsObjectAcd describes a acd object
+//
+// Will definitely have info but maybe not meta
+type FsObjectAcd struct {
+	acd    *FsAcd    // what this object is part of
+	remote string    // The remote path
+	info   *acd.Node // Info from the acd object if known
+}
+
+// ------------------------------------------------------------
+
+// The name of the remote (as passed into NewFs)
+func (f *FsAcd) Name() string {
+	return f.name
+}
+
+// The root of the remote (as passed into NewFs)
+func (f *FsAcd) Root() string {
+	return f.root
+}
+
+// String converts this FsAcd to a string
+func (f *FsAcd) String() string {
+	return fmt.Sprintf("Amazon cloud drive root '%s'", f.root)
+}
+
+// Pattern to match a acd path
+var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
+
+// parsePath parses an acd 'url'
+func parsePath(path string) (root string) {
+	root = strings.Trim(path, "/")
+	return
+}
+
+// NewFs contstructs an FsAcd from the path, container:path
+func NewFs(name, root string) (fs.Fs, error) {
+	root = parsePath(root)
+	oAuthClient, err := oauthutil.NewClient(name, acdConfig)
+	if err != nil {
+		log.Fatalf("Failed to configure amazon cloud drive: %v", err)
+	}
+
+	c := acd.NewClient(oAuthClient)
+	c.UserAgent = fs.UserAgent
+	f := &FsAcd{
+		name: name,
+		root: root,
+		c:    c,
+	}
+
+	// Update endpoints
+	_, _, err = f.c.Account.GetEndpoints()
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get endpoints: %v", err)
+	}
+
+	// Get rootID
+	rootInfo, _, err := f.c.Nodes.GetRoot()
+	if err != nil || rootInfo.Id == nil {
+		return nil, fmt.Errorf("Failed to get root: %v", err)
+	}
+
+	f.dirCache = dircache.New(root, *rootInfo.Id, f)
+
+	// Find the current root
+	err = f.dirCache.FindRoot(false)
+	if err != nil {
+		// Assume it is a file
+		newRoot, remote := dircache.SplitPath(root)
+		newF := *f
+		newF.dirCache = dircache.New(newRoot, *rootInfo.Id, &newF)
+		newF.root = newRoot
+		// Make new Fs which is the parent
+		err = newF.dirCache.FindRoot(false)
+		if err != nil {
+			// No root so return old f
+			return f, nil
+		}
+		obj := newF.newFsObjectWithInfo(remote, nil)
+		if obj == nil {
+			// File doesn't exist so return old f
+			return f, nil
+		}
+		// return a Fs Limited to this object
+		return fs.NewLimited(&newF, obj), nil
+	}
+	return f, nil
+}
+
+// Return an FsObject from a path
+//
+// May return nil if an error occurred
+func (f *FsAcd) newFsObjectWithInfo(remote string, info *acd.Node) fs.Object {
+	o := &FsObjectAcd{
+		acd:    f,
+		remote: remote,
+	}
+	if info != nil {
+		// Set info but not meta
+		o.info = info
+	} else {
+		err := o.readMetaData() // reads info and meta, returning an error
+		if err != nil {
+			// logged already FsDebug("Failed to read info: %s", err)
+			return nil
+		}
+	}
+	return o
+}
+
+// Return an FsObject from a path
+//
+// May return nil if an error occurred
+func (f *FsAcd) NewFsObject(remote string) fs.Object {
+	return f.newFsObjectWithInfo(remote, nil)
+}
+
+// FindLeaf finds a directory of name leaf in the folder with ID pathId
+func (f *FsAcd) FindLeaf(pathId, leaf string) (pathIdOut string, found bool, err error) {
+	//fs.Debug(f, "FindLeaf(%q, %q)", pathId, leaf)
+	folder := acd.FolderFromId(pathId, f.c.Nodes)
+	subFolder, _, err := folder.GetFolder(leaf)
+	if err != nil {
+		if err == acd.ErrorNodeNotFound {
+			//fs.Debug(f, "...Not found")
+			return "", false, nil
+		}
+		//fs.Debug(f, "...Error %v", err)
+		return "", false, err
+	}
+	if subFolder.Status != nil && *subFolder.Status != statusAvailable {
+		fs.Debug(f, "Ignoring folder %q in state %q", *subFolder.Status)
+		time.Sleep(1 * time.Second) // FIXME wait for problem to go away!
+		return "", false, nil
+	}
+	//fs.Debug(f, "...Found(%q, %v)", *subFolder.Id, leaf)
+	return *subFolder.Id, true, nil
+}
+
+// CreateDir makes a directory with pathId as parent and name leaf
+func (f *FsAcd) CreateDir(pathId, leaf string) (newId string, err error) {
+	//fs.Debug(f, "CreateDir(%q, %q)", pathId, leaf)
+	folder := acd.FolderFromId(pathId, f.c.Nodes)
+	info, _, err := folder.CreateFolder(leaf)
+	if err != nil {
+		fs.Debug(f, "...Error %v", err)
+		return "", err
+	}
+	//fs.Debug(f, "...Id %q", *info.Id)
+	return *info.Id, nil
+}
+
+// list the objects into the function supplied
+//
+// If directories is set it only sends directories
+// User function to process a File item from listAll
+//
+// Should return true to finish processing
+type listAllFn func(*acd.Node) bool
+
+// Lists the directory required calling the user function on each item found
+//
+// If the user fn ever returns true then it early exits with found = true
+func (f *FsAcd) listAll(dirId string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
+	query := "parents:" + dirId
+	if directoriesOnly {
+		query += " AND kind:" + folderKind
+	} else if filesOnly {
+		query += " AND kind:" + fileKind
+	} else {
+		// FIXME none of these work
+		//query += " AND kind:(" + fileKind + " OR " + folderKind + ")"
+		//query += " AND (kind:" + fileKind + " OR kind:" + folderKind + ")"
+	}
+	opts := acd.NodeListOptions{
+		Filters: query,
+	}
+	var nodes []*acd.Node
+OUTER:
+	for {
+		nodes, _, err = f.c.Nodes.GetNodes(&opts)
+		if err != nil {
+			fs.Stats.Error()
+			fs.ErrorLog(f, "Couldn't list files: %v", err)
+			break
+		}
+		if nodes == nil {
+			break
+		}
+		for _, node := range nodes {
+			if node.Name != nil && node.Id != nil && node.Kind != nil && node.Status != nil {
+				// Ignore nodes if not AVAILABLE
+				if *node.Status != statusAvailable {
+					continue
+				}
+				if fn(node) {
+					found = true
+					break OUTER
+				}
+			}
+		}
+	}
+	return
+}
+
+// Path should be directory path either "" or "path/"
+//
+// List the directory using a recursive list from the root
+//
+// This fetches the minimum amount of stuff but does more API calls
+// which makes it slow
+func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
+	var subError error
+	// Make the API request
+	_, err := f.listAll(dirId, "", false, false, func(node *acd.Node) bool {
+		// Recurse on directories
+		// FIXME should do this in parallel
+		// use a wg to sync then collect error
+		switch *node.Kind {
+		case folderKind:
+			subError = f.listDirRecursive(*node.Id, path+*node.Name+"/", out)
+			if subError != nil {
+				return true
+			}
+		case fileKind:
+			if fs := f.newFsObjectWithInfo(path+*node.Name, node); fs != nil {
+				out <- fs
+			}
+		default:
+			// ignore ASSET etc
+		}
+		return false
+	})
+	if err != nil {
+		return err
+	}
+	if subError != nil {
+		return subError
+	}
+	return nil
+}
+
+// Walk the path returning a channel of FsObjects
+func (f *FsAcd) List() fs.ObjectsChan {
+	out := make(fs.ObjectsChan, fs.Config.Checkers)
+	go func() {
+		defer close(out)
+		err := f.dirCache.FindRoot(false)
+		if err != nil {
+			fs.Stats.Error()
+			fs.ErrorLog(f, "Couldn't find root: %s", err)
+		} else {
+			err = f.listDirRecursive(f.dirCache.RootID(), "", out)
+			if err != nil {
+				fs.Stats.Error()
+				fs.ErrorLog(f, "List failed: %s", err)
+			}
+		}
+	}()
+	return out
+}
+
+// Lists the containers
+func (f *FsAcd) ListDir() fs.DirChan {
+	out := make(fs.DirChan, fs.Config.Checkers)
+	go func() {
+		defer close(out)
+		err := f.dirCache.FindRoot(false)
+		if err != nil {
+			fs.Stats.Error()
+			fs.ErrorLog(f, "Couldn't find root: %s", err)
+		} else {
+			_, err := f.listAll(f.dirCache.RootID(), "", true, false, func(item *acd.Node) bool {
+				dir := &fs.Dir{
+					Name:  *item.Name,
+					Bytes: -1,
+					Count: -1,
+				}
+				dir.When, _ = time.Parse(timeFormat, *item.ModifiedDate)
+				out <- dir
+				return false
+			})
+			if err != nil {
+				fs.Stats.Error()
+				fs.ErrorLog(f, "ListDir failed: %s", err)
+			}
+		}
+	}()
+	return out
+}
+
+// Put the object into the container
+//
+// Copy the reader in to the new object which is returned
+//
+// The new object may have been created if an error is returned
+func (f *FsAcd) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
+	// Temporary FsObject under construction
+	o := &FsObjectAcd{
+		acd:    f,
+		remote: remote,
+	}
+	leaf, directoryID, err := f.dirCache.FindPath(remote, true)
+	if err != nil {
+		return nil, err
+	}
+	folder := acd.FolderFromId(directoryID, o.acd.c.Nodes)
+	info, _, err := folder.Put(in, leaf)
+	if err != nil {
+		return nil, err
+	}
+	o.info = info.Node
+	return o, nil
+}
+
+// Mkdir creates the container if it doesn't exist
+func (f *FsAcd) Mkdir() error {
+	return f.dirCache.FindRoot(true)
+}
+
+// purgeCheck remotes the root directory, if check is set then it
+// refuses to do so if it has anything in
+func (f *FsAcd) purgeCheck(check bool) error {
+	if f.root == "" {
+		return fmt.Errorf("Can't purge root directory")
+	}
+	dc := f.dirCache
+	err := dc.FindRoot(false)
+	if err != nil {
+		return err
+	}
+	rootID := dc.RootID()
+
+	if check {
+		// check directory is empty
+		empty := true
+		_, err := f.listAll(rootID, "", false, false, func(node *acd.Node) bool {
+			switch *node.Kind {
+			case folderKind:
+				empty = false
+				return true
+			case fileKind:
+				empty = false
+				return true
+			default:
+				fs.Debug("Found ASSET %s", *node.Id)
+			}
+			return false
+		})
+		if err != nil {
+			return err
+		}
+		if !empty {
+			return fmt.Errorf("Directory not empty")
+		}
+	}
+
+	node := acd.NodeFromId(rootID, f.c.Nodes)
+	_, err = node.Trash()
+	if err != nil {
+		return err
+	}
+
+	f.dirCache.ResetRoot()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Rmdir deletes the root folder
+//
+// Returns an error if it isn't empty
+func (f *FsAcd) Rmdir() error {
+	return f.purgeCheck(true)
+}
+
+// Return the precision
+func (f *FsAcd) Precision() time.Duration {
+	return fs.ModTimeNotSupported
+}
+
+// Copy src to this remote using server side copy operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+//func (f *FsAcd) Copy(src fs.Object, remote string) (fs.Object, error) {
+// srcObj, ok := src.(*FsObjectAcd)
+// if !ok {
+// 	fs.Debug(src, "Can't copy - not same remote type")
+// 	return nil, fs.ErrorCantCopy
+// }
+// srcFs := srcObj.acd
+// _, err := f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
+// if err != nil {
+// 	return nil, err
+// }
+// return f.NewFsObject(remote), nil
+//}
+
+// Purge deletes all the files and the container
+//
+// Optional interface: Only implement this if you have a way of
+// deleting all the files quicker than just running Remove() on the
+// result of List()
+func (f *FsAcd) Purge() error {
+	return f.purgeCheck(false)
+}
+
+// ------------------------------------------------------------
+
+// Return the parent Fs
+func (o *FsObjectAcd) Fs() fs.Fs {
+	return o.acd
+}
+
+// Return a string version
+func (o *FsObjectAcd) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Return the remote path
+func (o *FsObjectAcd) Remote() string {
+	return o.remote
+}
+
+// Md5sum returns the Md5sum of an object returning a lowercase hex string
+func (o *FsObjectAcd) Md5sum() (string, error) {
+	if o.info.ContentProperties.Md5 != nil {
+		return *o.info.ContentProperties.Md5, nil
+	}
+	return "", nil
+}
+
+// Size returns the size of an object in bytes
+func (o *FsObjectAcd) Size() int64 {
+	return int64(*o.info.ContentProperties.Size)
+}
+
+// readMetaData gets the metadata if it hasn't already been fetched
+//
+// it also sets the info
+func (o *FsObjectAcd) readMetaData() (err error) {
+	if o.info != nil {
+		return nil
+	}
+	leaf, directoryID, err := o.acd.dirCache.FindPath(o.remote, false)
+	if err != nil {
+		return err
+	}
+	folder := acd.FolderFromId(directoryID, o.acd.c.Nodes)
+	info, _, err := folder.GetFile(leaf)
+	if err != nil {
+		fs.Debug(o, "Failed to read info: %s", err)
+		return err
+	}
+	o.info = info.Node
+	return nil
+}
+
+// ModTime returns the modification time of the object
+//
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+func (o *FsObjectAcd) ModTime() time.Time {
+	err := o.readMetaData()
+	if err != nil {
+		fs.Log(o, "Failed to read metadata: %s", err)
+		return time.Now()
+	}
+	modTime, err := time.Parse(timeFormat, *o.info.ModifiedDate)
+	if err != nil {
+		fs.Log(o, "Failed to read mtime from object: %s", err)
+		return time.Now()
+	}
+	return modTime
+}
+
+// Sets the modification time of the local fs object
+func (o *FsObjectAcd) SetModTime(modTime time.Time) {
+	// FIXME not implemented
+	return
+}
+
+// Is this object storable
+func (o *FsObjectAcd) Storable() bool {
+	return true
+}
+
+// Open an object for read
+func (o *FsObjectAcd) Open() (in io.ReadCloser, err error) {
+	file := acd.File{Node: o.info}
+	in, _, err = file.Open()
+	return in, err
+}
+
+// Update the object with the contents of the io.Reader, modTime and size
+//
+// The new object may have been created if an error is returned
+func (o *FsObjectAcd) Update(in io.Reader, modTime time.Time, size int64) error {
+	file := acd.File{Node: o.info}
+	info, _, err := file.Overwrite(in)
+	if err != nil {
+		return err
+	}
+	o.info = info.Node
+	return nil
+}
+
+// Remove an object
+func (o *FsObjectAcd) Remove() error {
+	_, err := o.info.Trash()
+	return err
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs     = (*FsAcd)(nil)
+	_ fs.Purger = (*FsAcd)(nil)
+	//	_ fs.Copier   = (*FsAcd)(nil)
+	//	_ fs.Mover    = (*FsAcd)(nil)
+	//	_ fs.DirMover = (*FsAcd)(nil)
+	_ fs.Object = (*FsObjectAcd)(nil)
+)

amazonclouddrive/amazonclouddrive_test.go

@@ -0,0 +1,56 @@
+// Test AmazonCloudDrive filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: make gen_tests
+package amazonclouddrive_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/amazonclouddrive"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*amazonclouddrive.FsObjectAcd)(nil))
+	fstests.RemoteName = "TestAmazonCloudDrive:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
+func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsCopy(t *testing.T)                { fstests.TestFsCopy(t) }
+func TestFsMove(t *testing.T)                { fstests.TestFsMove(t) }
+func TestFsDirMove(t *testing.T)             { fstests.TestFsDirMove(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
+func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

docs/content/about.md

@@ -1,8 +1,8 @@
 ---
 title: "Rclone"
-description: "rclone syncs files to and from Google Drive, S3, Swift, Cloudfiles, Dropbox and Google Cloud Storage."
+description: "rclone syncs files to and from Google Drive, S3, Swift, Cloudfiles, Dropbox, Google Cloud Storage and Amazon Cloud Drive."
 type: page
-date: "2014-07-17"
+date: "2015-09-06"
 groups: ["about"]
 ---
 
@@ -18,6 +18,7 @@ Rclone is a command line program to sync files and directories to and from
   * Openstack Swift / Rackspace cloud files / Memset Memstore
   * Dropbox
   * Google Cloud Storage
+  * Amazon Cloud Drive
   * The local filesystem
 
 Features

docs/content/amazonclouddrive.md

@@ -0,0 +1,104 @@
+---
+title: "Amazon Cloud Drive"
+description: "Rclone docs for Amazon Cloud Drive"
+date: "2015-09-06"
+---
+
+<i class="fa fa-google"></i> Amazon Cloud Drive
+-----------------------------------------
+
+Paths are specified as `remote:path`
+
+Paths may be as deep as required, eg `remote:directory/subdirectory`.
+
+The initial setup for Amazon cloud drive involves getting a token from
+Amazon which you need to do in your browser.  `rclone config` walks
+you through it.
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+n) New remote
+d) Delete remote
+q) Quit config
+e/n/d/q> n
+name> remote
+What type of source is it?
+Choose a number from below
+ 1) amazon cloud drive
+ 2) drive
+ 3) dropbox
+ 4) google cloud storage
+ 5) local
+ 6) s3
+ 7) swift
+type> 1
+Amazon Application Client Id - leave blank to use rclone's.
+client_id> 
+Amazon Application Client Secret - leave blank to use rclone's.
+client_secret> 
+Remote config
+If your browser doesn't open automatically go to the following link
+https://www.amazon.com/ap/oa?client_id=amzn1.application-oa2-client.xxxxxxxxxxxxxxx&redirect_uri=http%3A%2F%2F127.0.0.1%3A53682%2F&response_type=code&scope=clouddrive%3Aread_all+clouddrive%3Awrite&state=xxxxxxxxxxxxxxxxx
+Log in, then cut and paste the token that is returned from the browser here
+Enter verification code> xxxxxxxxxxxxxxxxxxxx
+--------------------
+[remote]
+client_id = 
+client_secret = 
+token = {"access_token":"xxxxxxxxxxxxxxxxxxxxxxx","token_type":"bearer","refresh_token":"xxxxxxxxxxxxxxxxxx","expiry":"2015-09-06T16:07:39.658438471+01:00"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Note that rclone runs a webserver on your local machine to collect the
+token as returned from Amazon. This is only run from the moment it
+opens your browser to the moment you cut and paste the verification
+code.  This is on `http://127.0.0.1:53682/` and this it may require
+you to unblock it temporarily if you are running a host firewall.
+
+Once configured you can then use `rclone` like this,
+
+List directories in top level of your Amazon cloud drive
+
+    rclone lsd remote:
+
+List all the files in your Amazon cloud drive
+
+    rclone ls remote:
+
+To copy a local directory to an Amazon cloud drive directory called backup
+
+    rclone copy /home/source remote:backup
+
+### Modified time and MD5SUMs ###
+
+Amazon cloud drive doesn't allow modification times to be changed via
+the API so these won't be accurate or used for syncing.
+
+It does store MD5SUMs so for a more accurate sync, you can use the
+`--checksum` flag.
+
+### Deleting files ###
+
+Any files you delete with rclone will end up in the trash.  Amazon
+don't provide an API to permanently delete files, nor to empty the
+trash, so you will have to do that with one of Amazon's apps or via
+the Amazon cloud drive website.
+
+### Limitations ###
+
+Note that Amazon cloud drive is case sensitive so you can't have a
+file called "Hello.doc" and one called "hello.doc".
+
+Amazon cloud drive has rate limiting so you may notice errors in the
+sync (429 errors).  rclone will automatically retry the sync up to 3
+times by default (see `--retries` flag) which should hopefully work
+around this problem.

docs/content/overview.md

@@ -0,0 +1,71 @@
+---
+title: "Overview of cloud storage systems"
+description: "Overview of cloud storage systems"
+type: page
+date: "2015-09-06"
+---
+
+# Overview of cloud storage systems #
+
+Each cloud storage system is slighly different.  Rclone attempts to
+provide a unified interface to them, but some underlying differences
+show through.
+
+## Features ##
+
+Here is an overview of the major features of each cloud storage system.
+
+| Name                   | MD5SUM  | ModTime | Case Sensitive | Duplicate Files |
+| ---------------------- |:-------:|:-------:|:--------------:|:---------------:|
+| Google Drive           | Yes     | Yes     | No             | Yes             |
+| Amazon S3              | Yes     | Yes     | No             | No              |
+| Openstack Swift        | Yes     | Yes     | No             | No              |
+| Dropbox                | No      | No      | Yes            | No              |
+| Google Cloud Storage   | Yes     | Yes     | No             | No              |
+| Amazon Cloud Drive     | Yes     | No      | Yes            | No              |
+| The local filesystem   | Yes     | Yes     | Depends        | No              |
+
+### MD5SUM ###
+
+The cloud storage system supports MD5SUMs of the objects.  This
+is used if available when transferring data as an integrity check and
+can be specifically used with the `--checksum` flag in syncs and in
+the `check` command.
+
+### ModTime ###
+
+The cloud storage system supports setting modification times on
+objects.  If it does then this enables a using the modification times
+as part of the sync.  If not then only the size will be checked by
+default, though the MD5SUM can be checked with the `--checksum` flag.
+
+All cloud storage systems support some kind of date on the object and
+these will be set when transferring from the cloud storage system.
+
+### Case Sensitive ###
+
+If a cloud storage systems is case sensitive then it is possible to
+have two files which differ only in case, eg `file.txt` and
+`FILE.txt`.  If a cloud storage system is case insensitive then that
+isn't possible.
+
+This can cause problems when syncing between a case insensitive
+system and a case sensitive system.  The symptom of this is that no
+matter how many times you run the sync it never completes fully.
+
+The local filesystem may or may not be case sensitive depending on OS.
+
+  * Windows - usuall case insensitive
+  * OSX - usually case insensitive, though it is possible to format case sensitive
+  * Linux - usually case sensitive, but there are case insensitive file systems (eg FAT formatted USB keys)
+
+Most of the time this doesn't cause any problems as people tend to
+avoid files whose name differs only by case even on case sensitive
+systems.
+
+### Duplicate files ###
+
+If a cloud storage system allows duplicate files then it can have two
+objects with the same name.
+
+This confuses rclone greatly when syncing.

docs/layouts/chrome/navbar.html

@@ -28,11 +28,13 @@
                 <li class="dropdown">
                   <a href="#" class="dropdown-toggle" data-toggle="dropdown"><b class="caret"></b> Storage Systems</a>
                   <ul class="dropdown-menu">
+                    <li><a href="/overview/"><i class="fa fa-archive"></i> Overview</a></li>
                     <li><a href="/drive/"><i class="fa fa-google"></i> Drive</a></li>
                     <li><a href="/s3/"><i class="fa fa-archive"></i> S3</a></li>
                     <li><a href="/swift/"><i class="fa fa-space-shuttle"></i> Swift</a></li>
                     <li><a href="/dropbox/"><i class="fa fa-dropbox"></i> Dropbox</a></li>
                     <li><a href="/googlecloudstorage/"><i class="fa fa-google"></i> Google Cloud Storage</a></li>
+                    <li><a href="/amazonclouddrive/"><i class="fa fa-archive"></i> Amazon Cloud Drive</a></li>
                     <li><a href="/local/"><i class="fa fa-file"></i> Local</a></li>
                   </ul>
                 </li>

docs/static/css/custom.css

@@ -4,4 +4,23 @@ body {
 
 footer {
     margin: 50px 0;
-}
+}
+
+table {
+   background-color:#e0e0ff
+}
+
+tbody td, th {
+    border: 1px solid black;
+    padding: 3px 7px 2px 7px;
+}
+
+thead td, th {
+    border: 1px solid black;
+    padding: 3px 7px 2px 7px;
+    font-weight: bold;
+}
+
+tbody tr:nth-child(odd) {
+   background-color:#d0d0ff
+}

fs/operations_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/ncw/rclone/fstest"
 
 	// Active file systems
+	_ "github.com/ncw/rclone/amazonclouddrive"
 	_ "github.com/ncw/rclone/drive"
 	_ "github.com/ncw/rclone/dropbox"
 	_ "github.com/ncw/rclone/googlecloudstorage"

fs/test_all.sh

@@ -8,6 +8,11 @@ TestS3:
 TestDrive:
 TestGoogleCloudStorage:
 TestDropbox:
+TestAmazonCloudDrive:
+"
+
+REMOTES="
+TestAmazonCloudDrive:
 "
 
 function test_remote {

fstest/fstests/gen_tests.go

@@ -75,17 +75,10 @@ func init() {
 `
 
 // Generate test file piping it through gofmt
-func generateTestProgram(t *template.Template, fns []string, Fsname string) {
+func generateTestProgram(t *template.Template, fns []string, Fsname, ObjectName string) {
 	fsname := strings.ToLower(Fsname)
 	TestName := "Test" + Fsname + ":"
 	outfile := "../../" + fsname + "/" + fsname + "_test.go"
-	// Find last capitalised group to be object name
-	matcher := regexp.MustCompile(`([A-Z][a-z0-9]+)$`)
-	matches := matcher.FindStringSubmatch(Fsname)
-	if len(matches) == 0 {
-		log.Fatalf("Couldn't find object name in %q", Fsname)
-	}
-	ObjectName := matches[1]
 
 	if fsname == "local" {
 		TestName = ""
@@ -133,11 +126,12 @@ func generateTestProgram(t *template.Template, fns []string, Fsname string) {
 func main() {
 	fns := findTestFunctions()
 	t := template.Must(template.New("main").Parse(testProgram))
-	generateTestProgram(t, fns, "Local")
+	generateTestProgram(t, fns, "Local", "Local")
-	generateTestProgram(t, fns, "Swift")
+	generateTestProgram(t, fns, "Swift", "Swift")
-	generateTestProgram(t, fns, "S3")
+	generateTestProgram(t, fns, "S3", "S3")
-	generateTestProgram(t, fns, "Drive")
+	generateTestProgram(t, fns, "Drive", "Drive")
-	generateTestProgram(t, fns, "GoogleCloudStorage")
+	generateTestProgram(t, fns, "GoogleCloudStorage", "Storage")
-	generateTestProgram(t, fns, "Dropbox")
+	generateTestProgram(t, fns, "Dropbox", "Dropbox")
+	generateTestProgram(t, fns, "AmazonCloudDrive", "Acd")
 	log.Printf("Done")
 }

make_manual.py

@@ -16,11 +16,13 @@ docs = [
     "about.md",
     "install.md",
     "docs.md",
+    "overview.md",
     "drive.md",
     "s3.md",
     "swift.md",
     "dropbox.md",
     "googlecloudstorage.md",
+    "amazonclouddrive.md",
     "local.md",
     "changelog.md",
     "bugs.md",

oauthutil/oauthutil.go

@@ -1,12 +1,16 @@
 package oauthutil
 
 import (
+	"crypto/rand"
 	"encoding/json"
 	"fmt"
+	"log"
+	"net"
 	"net/http"
 	"time"
 
 	"github.com/ncw/rclone/fs"
+	"github.com/skratchdot/open-golang/open"
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2"
 )
@@ -139,7 +143,9 @@ func NewClient(name string, config *oauth2.Config) (*http.Client, error) {
 }
 
 // Config does the initial creation of the token
-func Config(name string, config *oauth2.Config) error {
+//
+// It runs an internal webserver to receive the results
+func ConfigWithWebserver(name string, config *oauth2.Config, bindAddress string) error {
 	// See if already have a token
 	tokenString := fs.ConfigFile.MustValue(name, "token")
 	if tokenString != "" {
@@ -149,11 +155,30 @@ func Config(name string, config *oauth2.Config) error {
 		}
 	}
 
+	// Make random state
+	stateBytes := make([]byte, 16)
+	_, err := rand.Read(stateBytes)
+	if err != nil {
+		return err
+	}
+	state := fmt.Sprintf("%x", stateBytes)
+
+	// Prepare webserver
+	server := authServer{
+		state:       state,
+		bindAddress: bindAddress,
+	}
+	if bindAddress != "" {
+		go server.Start()
+		defer server.Stop()
+	}
+
 	// Generate a URL for the user to visit for authorization.
-	authUrl := config.AuthCodeURL("state")
+	authUrl := config.AuthCodeURL(state)
-	fmt.Printf("Go to the following link in your browser\n")
+	_ = open.Start(authUrl)
+	fmt.Printf("If your browser doesn't open automatically go to the following link\n")
 	fmt.Printf("%s\n", authUrl)
-	fmt.Printf("Log in, then type paste the token that is returned in the browser here\n")
+	fmt.Printf("Log in, then cut and paste the token that is returned from the browser here\n")
 
 	// Read the code, and exchange it for a token.
 	fmt.Printf("Enter verification code> ")
@@ -164,3 +189,60 @@ func Config(name string, config *oauth2.Config) error {
 	}
 	return putToken(name, token)
 }
+
+// Config does the initial creation of the token
+func Config(name string, config *oauth2.Config) error {
+	return ConfigWithWebserver(name, config, "")
+}
+
+// Local web server for collecting auth
+type authServer struct {
+	state       string
+	listener    net.Listener
+	bindAddress string
+}
+
+// startWebServer runs an internal web server to receive config details
+func (s *authServer) Start() {
+	fs.Debug(nil, "Starting auth server on %s", s.bindAddress)
+	mux := http.NewServeMux()
+	server := &http.Server{
+		Addr:    s.bindAddress,
+		Handler: mux,
+	}
+	mux.HandleFunc("/favicon.ico", func(w http.ResponseWriter, req *http.Request) {
+		http.Error(w, "", 404)
+		return
+	})
+	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
+		fs.Debug(nil, "Received request on auth server")
+		code := req.FormValue("code")
+		if code != "" {
+			state := req.FormValue("state")
+			if state != s.state {
+				fs.Debug(nil, "State did not match: want %q got %q", s.state, state)
+				fmt.Fprintf(w, "<h1>Failure</h1>\n<p>Auth state doesn't match</p>")
+			} else {
+				fs.Debug(nil, "Successfully got code")
+				fmt.Fprintf(w, "<h1>Success</h1>\n<p>Cut and paste this code into rclone: <code>%s</code></p>", code)
+			}
+			return
+		}
+		fs.Debug(nil, "No code found on request")
+		fmt.Fprintf(w, "<h1>Failed!</h1>\nNo code found.")
+		http.Error(w, "", 500)
+	})
+
+	var err error
+	s.listener, err = net.Listen("tcp", s.bindAddress)
+	if err != nil {
+		log.Fatalf("Failed to start auth webserver: %v", err)
+	}
+	server.Serve(s.listener)
+	fs.Debug(nil, "Closed auth server")
+}
+
+func (s *authServer) Stop() {
+	fs.Debug(nil, "Closing auth server")
+	_ = s.listener.Close()
+}

rclone.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/ncw/rclone/fs"
 	// Active file systems
+	_ "github.com/ncw/rclone/amazonclouddrive"
 	_ "github.com/ncw/rclone/drive"
 	_ "github.com/ncw/rclone/dropbox"
 	_ "github.com/ncw/rclone/googlecloudstorage"