From 2d8e75cab4b80f2b324649b1be0a8dfe4d5b685f Mon Sep 17 00:00:00 2001 From: Jacob McNamee Date: Sat, 2 Sep 2017 01:29:01 -0700 Subject: [PATCH] Implement --immutable option --- docs/content/docs.md | 20 ++++++++++++++++++++ fs/config.go | 3 +++ fs/fs.go | 1 + fs/operations.go | 9 +++++++-- fs/sync.go | 28 +++++++++++++++++----------- fs/sync_test.go | 33 +++++++++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 13 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index db55892a4..ad0f9408c 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -433,6 +433,26 @@ Normally rclone would skip any files that have the same modification time and are the same size (or have the same checksum if using `--checksum`). +### --immutable ### + +Treat source and destination files as immutable and disallow +modification. + +With this option set, files will be created and deleted as requested, +but existing files will never be updated. If an existing file does +not match between the source and destination, rclone will give the error +`Source and destination exist but do not match: immutable file modified`. + +Note that only commands which transfer files (e.g. `sync`, `copy`, +`move`) are affected by this behavior, and only modification is +disallowed. Files may still be deleted explicitly (e.g. `delete`, +`purge`) or implicitly (e.g. `sync`, `move`). Use `copy --immutable` +if it is desired to avoid deletion as well as modification. + +This can be useful as an additional layer of protection for immutable +or append-only data sets (notably backup archives), where modification +implies corruption and should not be propagated. + ### --log-file=FILE ### Log all of rclone's output to FILE. This is not active by default. diff --git a/fs/config.go b/fs/config.go index 67ff18f80..df01ea9d8 100644 --- a/fs/config.go +++ b/fs/config.go @@ -102,6 +102,7 @@ var ( bindAddr = StringP("bind", "", "", "Local address to bind to for outgoing connections, IPv4, IPv6 or name.") disableFeatures = StringP("disable", "", "", "Disable a comma separated list of features. Use help to see a list.") userAgent = StringP("user-agent", "", "rclone/"+Version, "Set the user-agent to a specified string. The default is rclone/ version") + immutable = BoolP("immutable", "", false, "Do not modify files. Fail if existing files have been modified.") streamingUploadCutoff = SizeSuffix(100 * 1024) logLevel = LogLevelNotice statsLogLevel = LogLevelInfo @@ -240,6 +241,7 @@ type ConfigInfo struct { TPSLimitBurst int BindAddr net.IP DisableFeatures []string + Immutable bool StreamingUploadCutoff SizeSuffix } @@ -379,6 +381,7 @@ func LoadConfig() { Config.UseListR = *useListR Config.TPSLimit = *tpsLimit Config.TPSLimitBurst = *tpsLimitBurst + Config.Immutable = *immutable Config.BufferSize = bufferSize Config.StreamingUploadCutoff = streamingUploadCutoff diff --git a/fs/fs.go b/fs/fs.go index 0bba2ed10..c336e9b38 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -50,6 +50,7 @@ var ( ErrorNotDeletingDirs = errors.New("not deleting directories as there were IO errors") ErrorCantMoveOverlapping = errors.New("can't move files on overlapping remotes") ErrorDirectoryNotEmpty = errors.New("directory not empty") + ErrorImmutableModified = errors.New("immutable file modified") ) // RegInfo provides information about a filesystem diff --git a/fs/operations.go b/fs/operations.go index 2a08f2d7a..13cc983b6 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -180,8 +180,13 @@ func equal(src ObjectInfo, dst Object, sizeOnly, checkSum bool) bool { if Config.DryRun { Logf(src, "Not updating modification time as --dry-run") } else { - // Size and hash the same but mtime different so update the - // mtime of the dst object here + // Size and hash the same but mtime different + // Error if objects are treated as immutable + if Config.Immutable { + Errorf(dst, "Timestamp mismatch between immutable objects") + return false + } + // Update the mtime of the dst object here err := dst.SetModTime(srcModTime) if err == ErrorCantSetModTime { Debugf(dst, "src and dst identical but can't set mod time without re-uploading") diff --git a/fs/sync.go b/fs/sync.go index 44b268850..d2dfe18ea 100644 --- a/fs/sync.go +++ b/fs/sync.go @@ -264,20 +264,26 @@ func (s *syncCopyMove) pairChecker(in ObjectPairChan, out ObjectPairChan, wg *sy // Check to see if can store this if src.Storable() { if NeedTransfer(pair.dst, pair.src) { - // If destination already exists, then we must move it into --backup-dir if required - if pair.dst != nil && s.backupDir != nil { - remoteWithSuffix := pair.dst.Remote() + s.suffix - overwritten, _ := s.backupDir.NewObject(remoteWithSuffix) - err := Move(s.backupDir, overwritten, remoteWithSuffix, pair.dst) - if err != nil { - s.processError(err) + // If files are treated as immutable, fail if destination exists and does not match + if Config.Immutable && pair.dst != nil { + Errorf(pair.dst, "Source and destination exist but do not match: immutable file modified") + s.processError(ErrorImmutableModified) + } else { + // If destination already exists, then we must move it into --backup-dir if required + if pair.dst != nil && s.backupDir != nil { + remoteWithSuffix := pair.dst.Remote() + s.suffix + overwritten, _ := s.backupDir.NewObject(remoteWithSuffix) + err := Move(s.backupDir, overwritten, remoteWithSuffix, pair.dst) + if err != nil { + s.processError(err) + } else { + // If successful zero out the dst as it is no longer there and copy the file + pair.dst = nil + out <- pair + } } else { - // If successful zero out the dst as it is no longer there and copy the file - pair.dst = nil out <- pair } - } else { - out <- pair } } else { // If moving need to delete the files we don't need to copy diff --git a/fs/sync_test.go b/fs/sync_test.go index 9143a0d3f..33b4fe20a 100644 --- a/fs/sync_test.go +++ b/fs/sync_test.go @@ -996,3 +996,36 @@ func TestSyncUTFNorm(t *testing.T) { file1.Path = file2.Path fstest.CheckItems(t, r.fremote, file1) } + +// Test --immutable +func TestSyncImmutable(t *testing.T) { + r := NewRun(t) + defer r.Finalise() + + fs.Config.Immutable = true + defer func() { fs.Config.Immutable = false }() + + // Create file on source + file1 := r.WriteFile("existing", "potato", t1) + fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.fremote) + + // Should succeed + fs.Stats.ResetCounters() + err := fs.Sync(r.fremote, r.flocal) + require.NoError(t, err) + fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.fremote, file1) + + // Modify file data and timestamp on source + file2 := r.WriteFile("existing", "tomato", t2) + fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.fremote, file1) + + // Should fail with ErrorImmutableModified and not modify local or remote files + fs.Stats.ResetCounters() + err = fs.Sync(r.fremote, r.flocal) + assert.EqualError(t, err, fs.ErrorImmutableModified.Error()) + fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.fremote, file1) +}