diff --git a/backend/local/clone_darwin.go b/backend/local/clone_darwin.go new file mode 100644 index 000000000..2dbff2c05 --- /dev/null +++ b/backend/local/clone_darwin.go @@ -0,0 +1,66 @@ +//go:build darwin && cgo + +// Package local provides a filesystem interface +package local + +import ( + "context" + "runtime" + + "github.com/go-darwin/apfs" + "github.com/rclone/rclone/fs" +) + +// Copy src to this remote using server-side copy operations. +// +// # This is stored with the remote path given +// +// # It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + if runtime.GOOS != "darwin" || f.opt.TranslateSymlinks { + return nil, fs.ErrorCantCopy + } + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't clone - not same remote type") + return nil, fs.ErrorCantCopy + } + + // Create destination + dstObj := f.newObject(remote) + err := dstObj.mkdirAll() + if err != nil { + return nil, err + } + + err = Clone(srcObj.path, f.localPath(remote)) + if err != nil { + return nil, err + } + fs.Debugf(remote, "server-side cloned!") + return f.NewObject(ctx, remote) +} + +// Clone uses APFS cloning if possible, otherwise falls back to copying (with full metadata preservation) +// note that this is closely related to unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) but not 100% identical +// https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.html +func Clone(src, dst string) error { + state := apfs.CopyFileStateAlloc() + defer func() { + if err := apfs.CopyFileStateFree(state); err != nil { + fs.Errorf(dst, "free state error: %v", err) + } + }() + cloned, err := apfs.CopyFile(src, dst, state, apfs.COPYFILE_CLONE) + fs.Debugf(dst, "isCloned: %v, error: %v", cloned, err) + return err +} + +// Check the interfaces are satisfied +var ( + _ fs.Copier = &Fs{} +) diff --git a/backend/union/union_internal_test.go b/backend/union/union_internal_test.go index b6f78ec8d..ccc130e8f 100644 --- a/backend/union/union_internal_test.go +++ b/backend/union/union_internal_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "runtime" "testing" "time" @@ -95,6 +96,12 @@ func TestMoveCopy(t *testing.T) { fLocal := unionFs.upstreams[0].Fs fMemory := unionFs.upstreams[1].Fs + if runtime.GOOS == "darwin" { + // need to disable as this test specifically tests a local that can't Copy + f.Features().Disable("Copy") + fLocal.Features().Disable("Copy") + } + t.Run("Features", func(t *testing.T) { assert.NotNil(t, f.Features().Move) assert.Nil(t, f.Features().Copy) diff --git a/fs/operations/copy_test.go b/fs/operations/copy_test.go index 73aa5e05a..1e8f3ae5e 100644 --- a/fs/operations/copy_test.go +++ b/fs/operations/copy_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path" + "runtime" "sort" "strings" "testing" @@ -449,6 +450,14 @@ func TestCopyFileMaxTransfer(t *testing.T) { ci.MaxTransfer = sizeCutoff ci.CutoffMode = fs.CutoffModeHard + if runtime.GOOS == "darwin" { + // disable server-side copies as they don't count towards transfer size stats + r.Flocal.Features().Disable("Copy") + if r.Fremote.Features().IsLocal { + r.Fremote.Features().Disable("Copy") + } + } + // file1: Show a small file gets transferred OK accounting.Stats(ctx).ResetCounters() err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 2d221e492..f1c07921e 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -1382,6 +1382,12 @@ func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) { r.CheckLocalItems(t, file1, file2) r.CheckRemoteItems(t) + if runtime.GOOS == "darwin" { + r.Flocal.Features().Disable("Copy") // macOS cloning is too fast for this test! + if r.Fremote.Features().IsLocal { + r.Fremote.Features().Disable("Copy") // macOS cloning is too fast for this test! + } + } accounting.GlobalStats().ResetCounters() // ctx = predictDstFromLogger(ctx) // not currently supported (but tests do pass for CutoffModeSoft) startTime := time.Now() @@ -2569,6 +2575,14 @@ func TestMaxTransfer(t *testing.T) { r.CheckLocalItems(t, file1, file2, file3) r.CheckRemoteItems(t) + if runtime.GOOS == "darwin" { + // disable server-side copies as they don't count towards transfer size stats + r.Flocal.Features().Disable("Copy") + if r.Fremote.Features().IsLocal { + r.Fremote.Features().Disable("Copy") + } + } + accounting.GlobalStats().ResetCounters() // ctx = predictDstFromLogger(ctx) // not currently supported diff --git a/go.mod b/go.mod index a29a24326..26f74b251 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/gabriel-vasile/mimetype v1.4.4 github.com/gdamore/tcell/v2 v2.7.4 github.com/go-chi/chi/v5 v5.1.0 + github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 github.com/go-git/go-billy/v5 v5.5.0 github.com/google/uuid v1.6.0 github.com/hanwen/go-fuse/v2 v2.5.1 diff --git a/go.sum b/go.sum index 1ade31d23..174178cd5 100644 --- a/go.sum +++ b/go.sum @@ -225,6 +225,8 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= +github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=