diff --git a/backend/cache/cache_test.go b/backend/cache/cache_test.go index 594149596..faf33e5d7 100644 --- a/backend/cache/cache_test.go +++ b/backend/cache/cache_test.go @@ -16,10 +16,11 @@ import ( // TestIntegration runs integration tests against the remote func TestIntegration(t *testing.T) { fstests.Run(t, &fstests.Opt{ - RemoteName: "TestCache:", - NilObject: (*cache.Object)(nil), - UnimplementableFsMethods: []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter"}, - UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata"}, - SkipInvalidUTF8: true, // invalid UTF-8 confuses the cache + RemoteName: "TestCache:", + NilObject: (*cache.Object)(nil), + UnimplementableFsMethods: []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter", "DirSetModTime", "MkdirMetadata"}, + UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata"}, + UnimplementableDirectoryMethods: []string{"Metadata", "SetMetadata", "SetModTime"}, + SkipInvalidUTF8: true, // invalid UTF-8 confuses the cache }) } diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go index 1867dfc4a..42f8aa16f 100644 --- a/backend/chunker/chunker.go +++ b/backend/chunker/chunker.go @@ -338,13 +338,18 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, // Note 2: features.Fill() points features.PutStream to our PutStream, // but features.Mask() will nullify it if wrappedFs does not have it. f.features = (&fs.Features{ - CaseInsensitive: true, - DuplicateFiles: true, - ReadMimeType: false, // Object.MimeType not supported - WriteMimeType: true, - BucketBased: true, - CanHaveEmptyDirectories: true, - ServerSideAcrossConfigs: true, + CaseInsensitive: true, + DuplicateFiles: true, + ReadMimeType: false, // Object.MimeType not supported + WriteMimeType: true, + BucketBased: true, + CanHaveEmptyDirectories: true, + ServerSideAcrossConfigs: true, + ReadDirMetadata: true, + WriteDirMetadata: true, + WriteDirSetModTime: true, + UserDirMetadata: true, + DirModTimeUpdatesOnWrite: true, }).Fill(ctx, f).Mask(ctx, baseFs).WrapsFs(f, baseFs) f.features.Disable("ListR") // Recursive listing may cause chunker skip files @@ -821,8 +826,7 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP } case fs.Directory: isSubdir[entry.Remote()] = true - wrapDir := fs.NewDirCopy(ctx, entry) - wrapDir.SetRemote(entry.Remote()) + wrapDir := fs.NewDirWrapper(entry.Remote(), entry) tempEntries = append(tempEntries, wrapDir) default: if f.opt.FailHard { @@ -1571,6 +1575,14 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { return f.base.Mkdir(ctx, dir) } +// MkdirMetadata makes the root directory of the Fs object +func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) { + if do := f.base.Features().MkdirMetadata; do != nil { + return do(ctx, dir, metadata) + } + return nil, fs.ErrorNotImplemented +} + // Rmdir removes the directory (container, bucket) if empty // // Return an error if it doesn't exist or isn't empty @@ -2557,6 +2569,7 @@ var ( _ fs.Mover = (*Fs)(nil) _ fs.DirMover = (*Fs)(nil) _ fs.DirSetModTimer = (*Fs)(nil) + _ fs.MkdirMetadataer = (*Fs)(nil) _ fs.PutUncheckeder = (*Fs)(nil) _ fs.PutStreamer = (*Fs)(nil) _ fs.CleanUpper = (*Fs)(nil) diff --git a/backend/combine/combine.go b/backend/combine/combine.go index a6f61de77..3fdceb128 100644 --- a/backend/combine/combine.go +++ b/backend/combine/combine.go @@ -222,18 +222,23 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs } // check features var features = (&fs.Features{ - CaseInsensitive: true, - DuplicateFiles: false, - ReadMimeType: true, - WriteMimeType: true, - CanHaveEmptyDirectories: true, - BucketBased: true, - SetTier: true, - GetTier: true, - ReadMetadata: true, - WriteMetadata: true, - UserMetadata: true, - PartialUploads: true, + CaseInsensitive: true, + DuplicateFiles: false, + ReadMimeType: true, + WriteMimeType: true, + CanHaveEmptyDirectories: true, + BucketBased: true, + SetTier: true, + GetTier: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, + ReadDirMetadata: true, + WriteDirMetadata: true, + WriteDirSetModTime: true, + UserDirMetadata: true, + DirModTimeUpdatesOnWrite: true, + PartialUploads: true, }).Fill(ctx, f) canMove := true for _, u := range f.upstreams { @@ -440,6 +445,32 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { return u.f.Mkdir(ctx, uRemote) } +// MkdirMetadata makes the root directory of the Fs object +func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) { + u, uRemote, err := f.findUpstream(dir) + if err != nil { + return nil, err + } + do := u.f.Features().MkdirMetadata + if do == nil { + return nil, fs.ErrorNotImplemented + } + newDir, err := do(ctx, uRemote, metadata) + if err != nil { + return nil, err + } + entries := fs.DirEntries{newDir} + entries, err = u.wrapEntries(ctx, entries) + if err != nil { + return nil, err + } + newDir, ok := entries[0].(fs.Directory) + if !ok { + return nil, fmt.Errorf("internal error: expecting %T to be fs.Directory", entries[0]) + } + return newDir, nil +} + // purge the upstream or fallback to a slow way func (u *upstream) purge(ctx context.Context, dir string) (err error) { if do := u.f.Features().Purge; do != nil { @@ -755,12 +786,11 @@ func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.D case fs.Object: entries[i] = u.newObject(x) case fs.Directory: - newDir := fs.NewDirCopy(ctx, x) - newPath, err := u.pathAdjustment.do(newDir.Remote()) + newPath, err := u.pathAdjustment.do(x.Remote()) if err != nil { return nil, err } - newDir.SetRemote(newPath) + newDir := fs.NewDirWrapper(newPath, x) entries[i] = newDir default: return nil, fmt.Errorf("unknown entry type %T", entry) @@ -1116,6 +1146,7 @@ var ( _ fs.PutUncheckeder = (*Fs)(nil) _ fs.MergeDirser = (*Fs)(nil) _ fs.DirSetModTimer = (*Fs)(nil) + _ fs.MkdirMetadataer = (*Fs)(nil) _ fs.CleanUpper = (*Fs)(nil) _ fs.OpenWriterAter = (*Fs)(nil) _ fs.FullObject = (*Object)(nil) diff --git a/backend/compress/compress.go b/backend/compress/compress.go index aab41c6e3..5fb52c013 100644 --- a/backend/compress/compress.go +++ b/backend/compress/compress.go @@ -183,18 +183,23 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, // the features here are ones we could support, and they are // ANDed with the ones from wrappedFs f.features = (&fs.Features{ - CaseInsensitive: true, - DuplicateFiles: false, - ReadMimeType: false, - WriteMimeType: false, - GetTier: true, - SetTier: true, - BucketBased: true, - CanHaveEmptyDirectories: true, - ReadMetadata: true, - WriteMetadata: true, - UserMetadata: true, - PartialUploads: true, + CaseInsensitive: true, + DuplicateFiles: false, + ReadMimeType: false, + WriteMimeType: false, + GetTier: true, + SetTier: true, + BucketBased: true, + CanHaveEmptyDirectories: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, + ReadDirMetadata: true, + WriteDirMetadata: true, + WriteDirSetModTime: true, + UserDirMetadata: true, + DirModTimeUpdatesOnWrite: true, + PartialUploads: true, }).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs) // We support reading MIME types no matter the wrapped fs f.features.ReadMimeType = true @@ -784,6 +789,14 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { return f.Fs.Mkdir(ctx, dir) } +// MkdirMetadata makes the root directory of the Fs object +func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) { + if do := f.Fs.Features().MkdirMetadata; do != nil { + return do(ctx, dir, metadata) + } + return nil, fs.ErrorNotImplemented +} + // Rmdir removes the directory (container, bucket) if empty // // Return an error if it doesn't exist or isn't empty @@ -1506,6 +1519,7 @@ var ( _ fs.Mover = (*Fs)(nil) _ fs.DirMover = (*Fs)(nil) _ fs.DirSetModTimer = (*Fs)(nil) + _ fs.MkdirMetadataer = (*Fs)(nil) _ fs.PutStreamer = (*Fs)(nil) _ fs.CleanUpper = (*Fs)(nil) _ fs.UnWrapper = (*Fs)(nil) diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go index 270e9ff6b..ea1fbccd1 100644 --- a/backend/crypt/crypt.go +++ b/backend/crypt/crypt.go @@ -263,19 +263,24 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, // the features here are ones we could support, and they are // ANDed with the ones from wrappedFs f.features = (&fs.Features{ - CaseInsensitive: !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff, - DuplicateFiles: true, - ReadMimeType: false, // MimeTypes not supported with crypt - WriteMimeType: false, - BucketBased: true, - CanHaveEmptyDirectories: true, - SetTier: true, - GetTier: true, - ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs, - ReadMetadata: true, - WriteMetadata: true, - UserMetadata: true, - PartialUploads: true, + CaseInsensitive: !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff, + DuplicateFiles: true, + ReadMimeType: false, // MimeTypes not supported with crypt + WriteMimeType: false, + BucketBased: true, + CanHaveEmptyDirectories: true, + SetTier: true, + GetTier: true, + ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, + ReadDirMetadata: true, + WriteDirMetadata: true, + WriteDirSetModTime: true, + UserDirMetadata: true, + DirModTimeUpdatesOnWrite: true, + PartialUploads: true, }).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs) return f, err @@ -520,6 +525,25 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { return f.Fs.Mkdir(ctx, f.cipher.EncryptDirName(dir)) } +// MkdirMetadata makes the root directory of the Fs object +func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) { + do := f.Fs.Features().MkdirMetadata + if do == nil { + return nil, fs.ErrorNotImplemented + } + newDir, err := do(ctx, f.cipher.EncryptDirName(dir), metadata) + if err != nil { + return nil, err + } + var entries = make(fs.DirEntries, 0, 1) + f.addDir(ctx, &entries, newDir) + newDir, ok := entries[0].(fs.Directory) + if !ok { + return nil, fmt.Errorf("internal error: expecting %T to be fs.Directory", entries[0]) + } + return newDir, nil +} + // DirSetModTime sets the directory modtime for dir func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error { do := f.Fs.Features().DirSetModTime @@ -770,7 +794,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error { } out := make([]fs.Directory, len(dirs)) for i, dir := range dirs { - out[i] = fs.NewDirCopy(ctx, dir).SetRemote(f.cipher.EncryptDirName(dir.Remote())) + out[i] = fs.NewDirWrapper(f.cipher.EncryptDirName(dir.Remote()), dir) } return do(ctx, out) } @@ -1006,14 +1030,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // newDir returns a dir with the Name decrypted func (f *Fs) newDir(ctx context.Context, dir fs.Directory) fs.Directory { - newDir := fs.NewDirCopy(ctx, dir) remote := dir.Remote() decryptedRemote, err := f.cipher.DecryptDirName(remote) if err != nil { fs.Debugf(remote, "Undecryptable dir name: %v", err) } else { - newDir.SetRemote(decryptedRemote) + remote = decryptedRemote } + newDir := fs.NewDirWrapper(remote, dir) return newDir } @@ -1217,6 +1241,7 @@ var ( _ fs.Wrapper = (*Fs)(nil) _ fs.MergeDirser = (*Fs)(nil) _ fs.DirSetModTimer = (*Fs)(nil) + _ fs.MkdirMetadataer = (*Fs)(nil) _ fs.DirCacheFlusher = (*Fs)(nil) _ fs.ChangeNotifier = (*Fs)(nil) _ fs.PublicLinker = (*Fs)(nil) diff --git a/backend/hasher/hasher.go b/backend/hasher/hasher.go index da320bdd0..f43c03896 100644 --- a/backend/hasher/hasher.go +++ b/backend/hasher/hasher.go @@ -164,16 +164,21 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs } stubFeatures := &fs.Features{ - CanHaveEmptyDirectories: true, - IsLocal: true, - ReadMimeType: true, - WriteMimeType: true, - SetTier: true, - GetTier: true, - ReadMetadata: true, - WriteMetadata: true, - UserMetadata: true, - PartialUploads: true, + CanHaveEmptyDirectories: true, + IsLocal: true, + ReadMimeType: true, + WriteMimeType: true, + SetTier: true, + GetTier: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, + ReadDirMetadata: true, + WriteDirMetadata: true, + WriteDirSetModTime: true, + UserDirMetadata: true, + DirModTimeUpdatesOnWrite: true, + PartialUploads: true, } f.features = stubFeatures.Fill(ctx, f).Mask(ctx, f.Fs).WrapsFs(f, f.Fs) @@ -349,6 +354,14 @@ func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) e return fs.ErrorNotImplemented } +// MkdirMetadata makes the root directory of the Fs object +func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) { + if do := f.Fs.Features().MkdirMetadata; do != nil { + return do(ctx, dir, metadata) + } + return nil, fs.ErrorNotImplemented +} + // DirCacheFlush resets the directory cache - used in testing // as an optional interface func (f *Fs) DirCacheFlush() { @@ -539,6 +552,7 @@ var ( _ fs.Wrapper = (*Fs)(nil) _ fs.MergeDirser = (*Fs)(nil) _ fs.DirSetModTimer = (*Fs)(nil) + _ fs.MkdirMetadataer = (*Fs)(nil) _ fs.DirCacheFlusher = (*Fs)(nil) _ fs.ChangeNotifier = (*Fs)(nil) _ fs.PublicLinker = (*Fs)(nil) diff --git a/backend/union/entry.go b/backend/union/entry.go index 24ec2f518..20a7d246c 100644 --- a/backend/union/entry.go +++ b/backend/union/entry.go @@ -27,6 +27,7 @@ type Object struct { // This is a wrapped object contains all candidates type Directory struct { *upstream.Directory + fs *Fs // what this directory is part of cd []upstream.Entry } @@ -227,7 +228,56 @@ func (d *Directory) Size() (s int64) { return s } +// SetMetadata sets metadata for an DirEntry +// +// It should return fs.ErrorNotImplemented if it can't set metadata +func (d *Directory) SetMetadata(ctx context.Context, metadata fs.Metadata) error { + entries, err := d.fs.actionEntries(d.candidates()...) + if err != nil { + return err + } + var wg sync.WaitGroup + errs := Errors(make([]error, len(entries))) + multithread(len(entries), func(i int) { + if d, ok := entries[i].(*upstream.Directory); ok { + err := d.SetMetadata(ctx, metadata) + if err != nil { + errs[i] = fmt.Errorf("%s: %w", d.UpstreamFs().Name(), err) + } + } else { + errs[i] = fs.ErrorIsFile + } + }) + wg.Wait() + return errs.Err() +} + +// SetModTime sets the metadata on the DirEntry to set the modification date +// +// If there is any other metadata it does not overwrite it. +func (d *Directory) SetModTime(ctx context.Context, t time.Time) error { + entries, err := d.fs.actionEntries(d.candidates()...) + if err != nil { + return err + } + var wg sync.WaitGroup + errs := Errors(make([]error, len(entries))) + multithread(len(entries), func(i int) { + if d, ok := entries[i].(*upstream.Directory); ok { + err := d.SetModTime(ctx, t) + if err != nil { + errs[i] = fmt.Errorf("%s: %w", d.UpstreamFs().Name(), err) + } + } else { + errs[i] = fs.ErrorIsFile + } + }) + wg.Wait() + return errs.Err() +} + // Check the interfaces are satisfied var ( - _ fs.FullObject = (*Object)(nil) + _ fs.FullObject = (*Object)(nil) + _ fs.FullDirectory = (*Directory)(nil) ) diff --git a/backend/union/union.go b/backend/union/union.go index a3cb1ce80..833ed1838 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -95,6 +95,7 @@ func (f *Fs) wrapEntries(entries ...upstream.Entry) (entry, error) { case *upstream.Directory: return &Directory{ Directory: e, + fs: f, cd: entries, }, nil default: @@ -182,6 +183,51 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { return err } +// MkdirMetadata makes the root directory of the Fs object +func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) { + upstreams, err := f.create(ctx, dir) + if err != nil { + return nil, err + } + errs := Errors(make([]error, len(upstreams))) + entries := make([]upstream.Entry, len(upstreams)) + multithread(len(upstreams), func(i int) { + u := upstreams[i] + if do := u.Features().MkdirMetadata; do != nil { + newDir, err := do(ctx, dir, metadata) + if err != nil { + errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err) + } else { + entries[i], err = u.WrapEntry(newDir) + if err != nil { + errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err) + } + } + + } else { + // Just do Mkdir on upstreams which don't support MkdirMetadata + err := u.Mkdir(ctx, dir) + if err != nil { + errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err) + } + } + }) + err = errs.Err() + if err != nil { + return nil, err + } + + entry, err := f.wrapEntries(entries...) + if err != nil { + return nil, err + } + newDir, ok := entry.(fs.Directory) + if !ok { + return nil, fmt.Errorf("internal error: expecting %T to be an fs.Directory", entry) + } + return newDir, nil +} + // Purge all files in the directory // // Implement this if you have a way of deleting all the files @@ -922,18 +968,23 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } fs.Debugf(f, "actionPolicy = %T, createPolicy = %T, searchPolicy = %T", f.actionPolicy, f.createPolicy, f.searchPolicy) var features = (&fs.Features{ - CaseInsensitive: true, - DuplicateFiles: false, - ReadMimeType: true, - WriteMimeType: true, - CanHaveEmptyDirectories: true, - BucketBased: true, - SetTier: true, - GetTier: true, - ReadMetadata: true, - WriteMetadata: true, - UserMetadata: true, - PartialUploads: true, + CaseInsensitive: true, + DuplicateFiles: false, + ReadMimeType: true, + WriteMimeType: true, + CanHaveEmptyDirectories: true, + BucketBased: true, + SetTier: true, + GetTier: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, + ReadDirMetadata: true, + WriteDirMetadata: true, + WriteDirSetModTime: true, + UserDirMetadata: true, + DirModTimeUpdatesOnWrite: true, + PartialUploads: true, }).Fill(ctx, f) canMove, slowHash := true, false for _, f := range upstreams { @@ -1009,6 +1060,7 @@ var ( _ fs.Mover = (*Fs)(nil) _ fs.DirMover = (*Fs)(nil) _ fs.DirSetModTimer = (*Fs)(nil) + _ fs.MkdirMetadataer = (*Fs)(nil) _ fs.DirCacheFlusher = (*Fs)(nil) _ fs.ChangeNotifier = (*Fs)(nil) _ fs.Abouter = (*Fs)(nil) diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index 07bd732d9..b86fa9797 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -322,6 +322,39 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) { return do.Metadata(ctx) } +// Metadata returns metadata for an DirEntry +// +// It should return nil if there is no Metadata +func (e *Directory) Metadata(ctx context.Context) (fs.Metadata, error) { + do, ok := e.Directory.(fs.Metadataer) + if !ok { + return nil, nil + } + return do.Metadata(ctx) +} + +// SetMetadata sets metadata for an DirEntry +// +// It should return fs.ErrorNotImplemented if it can't set metadata +func (e *Directory) SetMetadata(ctx context.Context, metadata fs.Metadata) error { + do, ok := e.Directory.(fs.SetMetadataer) + if !ok { + return fs.ErrorNotImplemented + } + return do.SetMetadata(ctx, metadata) +} + +// SetModTime sets the metadata on the DirEntry to set the modification date +// +// If there is any other metadata it does not overwrite it. +func (e *Directory) SetModTime(ctx context.Context, t time.Time) error { + do, ok := e.Directory.(fs.SetModTimer) + if !ok { + return fs.ErrorNotImplemented + } + return do.SetModTime(ctx, t) +} + // Writeback writes the object back and returns a new object // // If it returns nil, nil then the original object is OK @@ -457,5 +490,6 @@ func (f *Fs) updateUsageCore(lock bool) error { // Check the interfaces are satisfied var ( - _ fs.FullObject = (*Object)(nil) + _ fs.FullObject = (*Object)(nil) + _ fs.FullDirectory = (*Directory)(nil) )