From 2aa264b33cb098f37eaeea43a87feec36b391ad0 Mon Sep 17 00:00:00 2001 From: Lesmiscore Date: Sat, 1 Oct 2022 00:10:57 +0900 Subject: [PATCH] smb: backend to support SMB - fixes #2042 --- README.md | 1 + backend/all/all.go | 1 + backend/smb/connpool.go | 229 +++++++++ backend/smb/smb.go | 789 +++++++++++++++++++++++++++++++ backend/smb/smb_test.go | 17 + bin/make_manual.py | 1 + docs/content/_index.md | 1 + docs/content/docs.md | 1 + docs/content/overview.md | 2 + docs/content/smb.md | 231 +++++++++ docs/layouts/chrome/navbar.html | 1 + fstest/test_all/config.yaml | 3 + fstest/testserver/init.d/TestSMB | 28 ++ go.mod | 2 + go.sum | 5 + 15 files changed, 1312 insertions(+) create mode 100644 backend/smb/connpool.go create mode 100644 backend/smb/smb.go create mode 100644 backend/smb/smb_test.go create mode 100644 docs/content/smb.md create mode 100755 fstest/testserver/init.d/TestSMB diff --git a/README.md b/README.md index 6c0e35752..44c8f8552 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * Seafile [:page_facing_up:](https://rclone.org/seafile/) * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs) * SFTP [:page_facing_up:](https://rclone.org/sftp/) + * SMB / CIFS [:page_facing_up:](https://rclone.org/smb/) * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath) * Storj [:page_facing_up:](https://rclone.org/storj/) * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/) diff --git a/backend/all/all.go b/backend/all/all.go index 5dae4d37d..19b1e1df5 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -44,6 +44,7 @@ import ( _ "github.com/rclone/rclone/backend/sftp" _ "github.com/rclone/rclone/backend/sharefile" _ "github.com/rclone/rclone/backend/sia" + _ "github.com/rclone/rclone/backend/smb" _ "github.com/rclone/rclone/backend/storj" _ "github.com/rclone/rclone/backend/sugarsync" _ "github.com/rclone/rclone/backend/swift" diff --git a/backend/smb/connpool.go b/backend/smb/connpool.go new file mode 100644 index 000000000..6f6fe8232 --- /dev/null +++ b/backend/smb/connpool.go @@ -0,0 +1,229 @@ +package smb + +import ( + "context" + "fmt" + "net" + "sync/atomic" + "time" + + smb2 "github.com/hirochachacha/go-smb2" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/fshttp" +) + +// dial starts a client connection to the given SMB server. It is a +// convenience function that connects to the given network address, +// initiates the SMB handshake, and then sets up a Client. +func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) { + dialer := fshttp.NewDialer(ctx) + tconn, err := dialer.Dial(network, addr) + if err != nil { + return nil, err + } + + pass := "" + if f.opt.Pass != "" { + pass, err = obscure.Reveal(f.opt.Pass) + if err != nil { + return nil, err + } + } + + d := &smb2.Dialer{ + Initiator: &smb2.NTLMInitiator{ + User: f.opt.User, + Password: pass, + Domain: f.opt.Domain, + }, + } + + session, err := d.DialContext(ctx, tconn) + if err != nil { + return nil, err + } + + return &conn{ + smbSession: session, + conn: &tconn, + }, nil +} + +// conn encapsulates a SMB client and corresponding SMB client +type conn struct { + conn *net.Conn + smbSession *smb2.Session + smbShare *smb2.Share + shareName string +} + +// Closes the connection +func (c *conn) close() (err error) { + if c.smbShare != nil { + err = c.smbShare.Umount() + } + sessionLogoffErr := c.smbSession.Logoff() + if err != nil { + return err + } + return sessionLogoffErr +} + +// True if it's closed +func (c *conn) closed() bool { + var nopErr error + if c.smbShare != nil { + // stat the current directory + _, nopErr = c.smbShare.Stat(".") + } else { + // list the shares + _, nopErr = c.smbSession.ListSharenames() + } + return nopErr == nil +} + +// Show that we are using a SMB session +// +// Call removeSession() when done +func (f *Fs) addSession() { + atomic.AddInt32(&f.sessions, 1) +} + +// Show the SMB session is no longer in use +func (f *Fs) removeSession() { + atomic.AddInt32(&f.sessions, -1) +} + +// getSessions shows whether there are any sessions in use +func (f *Fs) getSessions() int32 { + return atomic.LoadInt32(&f.sessions) +} + +// Open a new connection to the SMB server. +func (f *Fs) newConnection(ctx context.Context, share string) (c *conn, err error) { + c, err = f.dial(ctx, "tcp", f.opt.Host+":"+f.opt.Port) + if err != nil { + return nil, fmt.Errorf("couldn't connect SMB: %w", err) + } + if share != "" { + // mount the specified share as well if user requested + c.smbShare, err = c.smbSession.Mount(share) + if err != nil { + _ = c.smbSession.Logoff() + return nil, fmt.Errorf("couldn't initialize SMB: %w", err) + } + c.smbShare = c.smbShare.WithContext(ctx) + } + return c, nil +} + +// Ensure the specified share is mounted or the session is unmounted +func (c *conn) mountShare(share string) (err error) { + if c.shareName == share { + return nil + } + if c.smbShare != nil { + err = c.smbShare.Umount() + c.smbShare = nil + } + if err != nil { + return + } + if share != "" { + c.smbShare, err = c.smbSession.Mount(share) + if err != nil { + return + } + } + c.shareName = share + return nil +} + +// Get a SMB connection from the pool, or open a new one +func (f *Fs) getConnection(ctx context.Context, share string) (c *conn, err error) { + accounting.LimitTPS(ctx) + f.poolMu.Lock() + for len(f.pool) > 0 { + c = f.pool[0] + f.pool = f.pool[1:] + err = c.mountShare(share) + if err == nil { + break + } + fs.Debugf(f, "Discarding unusable SMB connection: %v", err) + c = nil + } + f.poolMu.Unlock() + if c != nil { + return c, nil + } + err = f.pacer.Call(func() (bool, error) { + c, err = f.newConnection(ctx, share) + if err != nil { + return true, err + } + return false, nil + }) + return c, err +} + +// Return a SMB connection to the pool +// +// It nils the pointed to connection out so it can't be reused +func (f *Fs) putConnection(pc **conn) { + c := *pc + *pc = nil + + var nopErr error + if c.smbShare != nil { + // stat the current directory + _, nopErr = c.smbShare.Stat(".") + } else { + // list the shares + _, nopErr = c.smbSession.ListSharenames() + } + if nopErr != nil { + fs.Debugf(f, "Connection failed, closing: %v", nopErr) + _ = c.close() + return + } + + f.poolMu.Lock() + f.pool = append(f.pool, c) + if f.opt.IdleTimeout > 0 { + f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer + } + f.poolMu.Unlock() +} + +// Drain the pool of any connections +func (f *Fs) drainPool(ctx context.Context) (err error) { + f.poolMu.Lock() + defer f.poolMu.Unlock() + if sessions := f.getSessions(); sessions != 0 { + fs.Debugf(f, "Not closing %d unused connections as %d sessions active", len(f.pool), sessions) + if f.opt.IdleTimeout > 0 { + f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer + } + return nil + } + if f.opt.IdleTimeout > 0 { + f.drain.Stop() + } + if len(f.pool) != 0 { + fs.Debugf(f, "Closing %d unused connections", len(f.pool)) + } + for i, c := range f.pool { + if !c.closed() { + cErr := c.close() + if cErr != nil { + err = cErr + } + } + f.pool[i] = nil + } + f.pool = nil + return err +} diff --git a/backend/smb/smb.go b/backend/smb/smb.go new file mode 100644 index 000000000..3e750cc1f --- /dev/null +++ b/backend/smb/smb.go @@ -0,0 +1,789 @@ +// Package smb provides an interface to SMB servers +package smb + +import ( + "context" + "fmt" + "io" + "os" + "path" + "strings" + "sync" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/bucket" + "github.com/rclone/rclone/lib/encoder" + "github.com/rclone/rclone/lib/env" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/readers" +) + +const ( + minSleep = 100 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential +) + +var ( + currentUser = env.CurrentUser() +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "smb", + Description: "SMB / CIFS", + NewFs: NewFs, + + Options: []fs.Option{{ + Name: "host", + Help: "SMB server hostname to connect to.\n\nE.g. \"example.com\".", + Required: true, + }, { + Name: "user", + Help: "SMB username.", + Default: currentUser, + }, { + Name: "port", + Help: "SMB port number.", + Default: 445, + }, { + Name: "pass", + Help: "SMB password.", + IsPassword: true, + }, { + Name: "domain", + Help: "Domain name for NTLM authentication.", + Default: "WORKGROUP", + }, { + Name: "idle_timeout", + Default: fs.Duration(60 * time.Second), + Help: `Max time before closing idle connections. + +If no connections have been returned to the connection pool in the time +given, rclone will empty the connection pool. + +Set to 0 to keep connections indefinitely. +`, + Advanced: true, + }, { + Name: "hide_special_share", + Help: "Hide special shares (e.g. print$) which users aren't supposed to access.", + Default: true, + Advanced: true, + }, { + Name: "case_insensitive", + Help: "Whether the server is configured to be case-insensitive.\n\nAlways true on Windows shares.", + Default: true, + Advanced: true, + }, { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + Default: encoder.EncodeZero | + // path separator + encoder.EncodeSlash | + encoder.EncodeBackSlash | + // windows + encoder.EncodeWin | + encoder.EncodeCtl | + encoder.EncodeDot | + // the file turns into 8.3 names (and cannot be converted back) + encoder.EncodeRightSpace | + encoder.EncodeRightPeriod | + // + encoder.EncodeInvalidUtf8, + }, + }}) +} + +// Options defines the configuration for this backend +type Options struct { + Host string `config:"host"` + Port string `config:"port"` + User string `config:"user"` + Pass string `config:"pass"` + Domain string `config:"domain"` + HideSpecial bool `config:"hide_special_share"` + CaseInsensitive bool `config:"case_insensitive"` + IdleTimeout fs.Duration `config:"idle_timeout"` + + Enc encoder.MultiEncoder `config:"encoding"` +} + +// Fs represents a SMB remote +type Fs struct { + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed config options + features *fs.Features // optional features + pacer *fs.Pacer // pacer for operations + + sessions int32 + poolMu sync.Mutex + pool []*conn + drain *time.Timer // used to drain the pool when we stop using the connections + + ctx context.Context +} + +// Object describes a file at the server +type Object struct { + fs *Fs // reference to Fs + remote string // the remote path + statResult os.FileInfo +} + +// NewFs constructs an Fs from the path +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + root = strings.Trim(root, "/") + + f := &Fs{ + name: name, + opt: *opt, + ctx: ctx, + root: root, + } + f.features = (&fs.Features{ + CaseInsensitive: opt.CaseInsensitive, + CanHaveEmptyDirectories: true, + BucketBased: true, + }).Fill(ctx, f) + + f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))) + // set the pool drainer timer going + if opt.IdleTimeout > 0 { + f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) }) + } + + // test if the root exists as a file + share, dir := f.split("") + if share == "" || dir == "" { + return f, nil + } + cn, err := f.getConnection(ctx, share) + if err != nil { + return nil, err + } + stat, err := cn.smbShare.Stat(f.toSambaPath(dir)) + f.putConnection(&cn) + if err != nil { + // ignore stat error here + return f, nil + } + if !stat.IsDir() { + f.root, err = path.Dir(root), fs.ErrorIsFile + } + fs.Debugf(f, "Using root directory %q", f.root) + return f, err +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + bucket, file := f.split("") + if bucket == "" { + return fmt.Sprintf("smb://%s@%s:%s/", f.opt.User, f.opt.Host, f.opt.Port) + } + return fmt.Sprintf("smb://%s@%s:%s/%s/%s", f.opt.User, f.opt.Host, f.opt.Port, bucket, file) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Hashes returns nothing as SMB itself doesn't have a way to tell checksums +func (f *Fs) Hashes() hash.Set { + return hash.NewHashSet() +} + +// Precision returns the precision of mtime +func (f *Fs) Precision() time.Duration { + return time.Millisecond +} + +// NewObject creates a new file object +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + share, path := f.split(remote) + return f.findObjectSeparate(ctx, share, path) +} + +func (f *Fs) findObjectSeparate(ctx context.Context, share, path string) (fs.Object, error) { + if share == "" || path == "" { + return nil, fs.ErrorIsDir + } + cn, err := f.getConnection(ctx, share) + if err != nil { + return nil, err + } + stat, err := cn.smbShare.Stat(f.toSambaPath(path)) + f.putConnection(&cn) + if err != nil { + return nil, translateError(err, false) + } + if stat.IsDir() { + return nil, fs.ErrorIsDir + } + + return f.makeEntry(share, path, stat), nil +} + +// Mkdir creates a directory on the server +func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) { + share, path := f.split(dir) + if share == "" || path == "" { + return nil + } + cn, err := f.getConnection(ctx, share) + if err != nil { + return err + } + err = cn.smbShare.MkdirAll(f.toSambaPath(path), 0o755) + f.putConnection(&cn) + return err +} + +// Rmdir removes an empty directory on the server +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + share, path := f.split(dir) + if share == "" || path == "" { + return nil + } + cn, err := f.getConnection(ctx, share) + if err != nil { + return err + } + err = cn.smbShare.Remove(f.toSambaPath(path)) + f.putConnection(&cn) + return err +} + +// Put uploads a file +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + o := &Object{ + fs: f, + remote: src.Remote(), + } + + err := o.Update(ctx, in, src, options...) + if err == nil { + return o, nil + } + + return nil, err +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + o := &Object{ + fs: f, + remote: src.Remote(), + } + + err := o.Update(ctx, in, src, options...) + if err == nil { + return o, nil + } + + return nil, err +} + +// Move src to this remote using server-side move operations. +// +// This is stored with the remote path given. +// +// It returns the destination Object and a possible error. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (_ fs.Object, err error) { + dstShare, dstPath := f.split(remote) + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + srcShare, srcPath := srcObj.split() + if dstShare != srcShare { + fs.Debugf(src, "Can't move - must be on the same share") + return nil, fs.ErrorCantMove + } + + err = f.ensureDirectory(ctx, dstShare, dstPath) + if err != nil { + return nil, fmt.Errorf("failed to make parent directories: %w", err) + } + + cn, err := f.getConnection(ctx, dstShare) + if err != nil { + return nil, err + } + err = cn.smbShare.Rename(f.toSambaPath(srcPath), f.toSambaPath(dstPath)) + f.putConnection(&cn) + if err != nil { + return nil, translateError(err, false) + } + return f.findObjectSeparate(ctx, dstShare, dstPath) +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server-side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) { + dstShare, dstPath := f.split(dstRemote) + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return fs.ErrorCantDirMove + } + srcShare, srcPath := srcFs.split(srcRemote) + if dstShare != srcShare { + fs.Debugf(src, "Can't move - must be on the same share") + return fs.ErrorCantDirMove + } + + err = f.ensureDirectory(ctx, dstShare, dstPath) + if err != nil { + return fmt.Errorf("failed to make parent directories: %w", err) + } + + cn, err := f.getConnection(ctx, dstShare) + if err != nil { + return err + } + defer f.putConnection(&cn) + + _, err = cn.smbShare.Stat(dstPath) + if os.IsNotExist(err) { + err = cn.smbShare.Rename(f.toSambaPath(srcPath), f.toSambaPath(dstPath)) + return translateError(err, true) + } + return fs.ErrorDirExists +} + +// List files and directories in a directory +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + share, _path := f.split(dir) + + cn, err := f.getConnection(ctx, share) + if err != nil { + return nil, err + } + defer f.putConnection(&cn) + + if share == "" { + shares, err := cn.smbSession.ListSharenames() + for _, shh := range shares { + shh = f.toNativePath(shh) + if strings.HasSuffix(shh, "$") && f.opt.HideSpecial { + continue + } + entries = append(entries, fs.NewDir(shh, time.Time{})) + } + return entries, err + } + + dirents, err := cn.smbShare.ReadDir(f.toSambaPath(_path)) + if err != nil { + return entries, translateError(err, true) + } + for _, file := range dirents { + nfn := f.toNativePath(file.Name()) + if file.IsDir() { + entries = append(entries, fs.NewDir(path.Join(dir, nfn), file.ModTime())) + } else { + entries = append(entries, f.makeEntryRelative(share, _path, nfn, file)) + } + } + + return entries, nil +} + +// About returns things about remaining and used spaces +func (f *Fs) About(ctx context.Context) (_ *fs.Usage, err error) { + share, dir := f.split("/") + if share == "" { + return nil, fs.ErrorListBucketRequired + } + dir = f.toSambaPath(dir) + + cn, err := f.getConnection(ctx, share) + if err != nil { + return nil, err + } + stat, err := cn.smbShare.Statfs(dir) + f.putConnection(&cn) + if err != nil { + return nil, err + } + + bs := int64(stat.BlockSize()) + usage := &fs.Usage{ + Total: fs.NewUsageValue(bs * int64(stat.TotalBlockCount())), + Used: fs.NewUsageValue(bs * int64(stat.TotalBlockCount()-stat.FreeBlockCount())), + Free: fs.NewUsageValue(bs * int64(stat.AvailableBlockCount())), + } + return usage, nil +} + +// Shutdown the backend, closing any background tasks and any +// cached connections. +func (f *Fs) Shutdown(ctx context.Context) error { + return f.drainPool(ctx) +} + +func (f *Fs) makeEntry(share, _path string, stat os.FileInfo) *Object { + remote := path.Join(share, _path) + return &Object{ + fs: f, + remote: trimPathPrefix(remote, f.root), + statResult: stat, + } +} + +func (f *Fs) makeEntryRelative(share, _path, relative string, stat os.FileInfo) *Object { + return f.makeEntry(share, path.Join(_path, relative), stat) +} + +func (f *Fs) ensureDirectory(ctx context.Context, share, _path string) error { + cn, err := f.getConnection(ctx, share) + if err != nil { + return err + } + err = cn.smbShare.MkdirAll(f.toSambaPath(path.Dir(_path)), 0o755) + f.putConnection(&cn) + return err +} + +/// Object + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// ModTime is the last modified time (read-only) +func (o *Object) ModTime(ctx context.Context) time.Time { + return o.statResult.ModTime() +} + +// Size is the file length +func (o *Object) Size() int64 { + return o.statResult.Size() +} + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Hash always returns empty value +func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// Storable returns if this object is storable +func (o *Object) Storable() bool { + return true +} + +// SetModTime sets modTime on a particular file +func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) { + share, reqDir := o.split() + if share == "" || reqDir == "" { + return fs.ErrorCantSetModTime + } + reqDir = o.fs.toSambaPath(reqDir) + + cn, err := o.fs.getConnection(ctx, share) + if err != nil { + return err + } + defer o.fs.putConnection(&cn) + + err = cn.smbShare.Chtimes(reqDir, t, t) + if err != nil { + return err + } + + fi, err := cn.smbShare.Stat(reqDir) + if err == nil { + o.statResult = fi + } + return err +} + +// Open an object for read +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + share, filename := o.split() + if share == "" || filename == "" { + return nil, fs.ErrorIsDir + } + filename = o.fs.toSambaPath(filename) + + var offset, limit int64 = 0, -1 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + case *fs.RangeOption: + offset, limit = x.Decode(o.Size()) + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + + o.fs.addSession() // Show session in use + defer o.fs.removeSession() + + cn, err := o.fs.getConnection(ctx, share) + if err != nil { + return nil, err + } + fl, err := cn.smbShare.OpenFile(filename, os.O_RDONLY, 0) + if err != nil { + o.fs.putConnection(&cn) + return nil, fmt.Errorf("failed to open: %w", err) + } + pos, err := fl.Seek(offset, io.SeekStart) + if err != nil { + o.fs.putConnection(&cn) + return nil, fmt.Errorf("failed to seek: %w", err) + } + if pos != offset { + o.fs.putConnection(&cn) + return nil, fmt.Errorf("failed to seek: wrong position (expected=%d, reported=%d)", offset, pos) + } + + in = readers.NewLimitedReadCloser(fl, limit) + in = &boundReadCloser{ + rc: in, + close: func() error { + o.fs.putConnection(&cn) + return nil + }, + } + + return in, nil +} + +// Update the Object from in with modTime and size +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + share, filename := o.split() + if share == "" || filename == "" { + return fs.ErrorIsDir + } + + err = o.fs.ensureDirectory(ctx, share, filename) + if err != nil { + return fmt.Errorf("failed to make parent directories: %w", err) + } + + filename = o.fs.toSambaPath(filename) + + o.fs.addSession() // Show session in use + defer o.fs.removeSession() + + cn, err := o.fs.getConnection(ctx, share) + if err != nil { + return err + } + defer func() { + o.statResult, _ = cn.smbShare.Stat(filename) + o.fs.putConnection(&cn) + }() + + fl, err := cn.smbShare.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("failed to open: %w", err) + } + + // remove the file if upload failed + remove := func() { + // Windows doesn't allow removal of files without closing file + removeErr := fl.Close() + if removeErr != nil { + fs.Debugf(src, "failed to close the file for delete: %v", removeErr) + // try to remove the file anyway; the file may be already closed + } + + removeErr = cn.smbShare.Remove(filename) + if removeErr != nil { + fs.Debugf(src, "failed to remove: %v", removeErr) + } else { + fs.Debugf(src, "removed after failed upload: %v", err) + } + } + + _, err = fl.ReadFrom(in) + if err != nil { + remove() + return fmt.Errorf("Update ReadFrom failed: %w", err) + } + + err = fl.Close() + if err != nil { + remove() + return fmt.Errorf("Update Close failed: %w", err) + } + + // Set the modified time + err = o.SetModTime(ctx, src.ModTime(ctx)) + if err != nil { + return fmt.Errorf("Update SetModTime failed: %w", err) + } + + return nil +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) (err error) { + share, filename := o.split() + if share == "" || filename == "" { + return fs.ErrorIsDir + } + filename = o.fs.toSambaPath(filename) + + cn, err := o.fs.getConnection(ctx, share) + if err != nil { + return err + } + + err = cn.smbShare.Remove(filename) + o.fs.putConnection(&cn) + + return err +} + +// String converts this Object to a string +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +/// Misc + +// split returns share name and path in the share from the rootRelativePath +// relative to f.root +func (f *Fs) split(rootRelativePath string) (shareName, filepath string) { + return bucket.Split(path.Join(f.root, rootRelativePath)) +} + +// split returns share name and path in the share from the object +func (o *Object) split() (shareName, filepath string) { + return o.fs.split(o.remote) +} + +func (f *Fs) toSambaPath(path string) string { + // 1. encode via Rclone's escaping system + // 2. convert to backslash-separated path + return strings.ReplaceAll(f.opt.Enc.FromStandardPath(path), "/", "\\") +} + +func (f *Fs) toNativePath(path string) string { + // 1. convert *back* to slash-separated path + // 2. encode via Rclone's escaping system + return f.opt.Enc.ToStandardPath(strings.ReplaceAll(path, "\\", "/")) +} + +func ensureSuffix(s, suffix string) string { + if strings.HasSuffix(s, suffix) { + return s + } + return s + suffix +} + +func trimPathPrefix(s, prefix string) string { + // we need to clean the paths to make tests pass! + s = betterPathClean(s) + prefix = betterPathClean(prefix) + if s == prefix || s == prefix+"/" { + return "" + } + prefix = ensureSuffix(prefix, "/") + return strings.TrimPrefix(s, prefix) +} + +func betterPathClean(p string) string { + d := path.Clean(p) + if d == "." { + return "" + } + return d +} + +type boundReadCloser struct { + rc io.ReadCloser + close func() error +} + +func (r *boundReadCloser) Read(p []byte) (n int, err error) { + return r.rc.Read(p) +} + +func (r *boundReadCloser) Close() error { + err1 := r.rc.Close() + err2 := r.close() + if err1 != nil { + return err1 + } + return err2 +} + +func translateError(e error, dir bool) error { + if os.IsNotExist(e) { + if dir { + return fs.ErrorDirNotFound + } + return fs.ErrorObjectNotFound + } + + return e +} + +var ( + _ fs.Fs = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.Mover = &Fs{} + _ fs.DirMover = &Fs{} + _ fs.Abouter = &Fs{} + _ fs.Shutdowner = &Fs{} + _ fs.Object = &Object{} + _ io.ReadCloser = &boundReadCloser{} +) diff --git a/backend/smb/smb_test.go b/backend/smb/smb_test.go new file mode 100644 index 000000000..aa5a0e419 --- /dev/null +++ b/backend/smb/smb_test.go @@ -0,0 +1,17 @@ +// Test smb filesystem interface +package smb_test + +import ( + "testing" + + "github.com/rclone/rclone/backend/smb" + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestSMB:rclone", + NilObject: (*smb.Object)(nil), + }) +} diff --git a/bin/make_manual.py b/bin/make_manual.py index fa14b36a3..cc2e81a5e 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -68,6 +68,7 @@ docs = [ "putio.md", "seafile.md", "sftp.md", + "smb.md", "storj.md", "sugarsync.md", "tardigrade.md", # stub only to redirect to storj.md diff --git a/docs/content/_index.md b/docs/content/_index.md index 98bcd5784..5d1bf2cda 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -162,6 +162,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}} {{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol" config="/sftp/" >}} {{< provider name="Sia" home="https://sia.tech/" config="/sia/" >}} +{{< provider name="SMB / CIFS" home="https://en.wikipedia.org/wiki/Server_Message_Block" config="/smb/" >}} {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}} {{< provider name="Storj" home="https://storj.io/" config="/storj/" >}} {{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index 47757d431..7f3b20378 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -69,6 +69,7 @@ See the following for detailed instructions for * [Seafile](/seafile/) * [SFTP](/sftp/) * [Sia](/sia/) + * [SMB](/smb/) * [Storj](/storj/) * [SugarSync](/sugarsync/) * [Union](/union/) diff --git a/docs/content/overview.md b/docs/content/overview.md index e20543238..34f059eb3 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -50,6 +50,7 @@ Here is an overview of the major features of each cloud storage system. | Seafile | - | - | No | No | - | - | | SFTP | MD5, SHA1 ² | R/W | Depends | No | - | - | | Sia | - | - | No | No | - | - | +| SMB | - | - | Yes | No | - | - | | SugarSync | - | - | No | No | - | - | | Storj | - | R | No | No | - | - | | Uptobox | - | - | No | Yes | - | - | @@ -501,6 +502,7 @@ upon backend-specific capabilities. | Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | | SFTP | No | No | Yes | Yes | No | No | Yes | No | Yes | Yes | | Sia | No | No | No | No | No | No | Yes | No | No | Yes | +| SMB | No | No | Yes | Yes | No | No | Yes | No | No | Yes | | SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | | Storj | Yes † | No | Yes | No | No | Yes | Yes | No | No | No | | Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | diff --git a/docs/content/smb.md b/docs/content/smb.md new file mode 100644 index 000000000..62c1717bb --- /dev/null +++ b/docs/content/smb.md @@ -0,0 +1,231 @@ +--- +title: "SMB / CIFS" +description: "Rclone docs for SMB backend" +--- + +# {{< icon "fa fa-server" >}} SMB + +SMB is [a communication protocol to share files over network](https://en.wikipedia.org/wiki/Server_Message_Block). + +This relies on [go-smb2 library](https://github.com/hirochachacha/go-smb2/) for communication with SMB protocol. + +Paths are specified as `remote:sharename` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`. + +## Notes + +The first path segment must be the name of the share, which you entered when you started to share on Windows. On smbd, it's the section title in `smb.conf` (usually in `/etc/samba/`) file. +You can find shares by quering the root if you're unsure (e.g. `rclone lsd remote:`). + +You can't access to the shared printers from rclone, obviously. + +You can't use Anonymous access for logging in. You have to use the `guest` user with an empty password instead. +The rclone client tries to avoid 8.3 names when uploading files by encoding trailing spaces and periods. +Alternatively, [the local backend](/local/#paths-on-windows) on Windows can access SMB servers using UNC paths, by `\\server\share`. This doesn't apply to non-Windows OSes, such as Linux and macOS. + +## Configuration + +Here is an example of making a SMB configuration. + +First run + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +XX / SMB / CIFS + \ (smb) +Storage> smb + +Option host. +Samba hostname to connect to. +E.g. "example.com". +Enter a value. +host> localhost + +Option user. +Samba username. +Enter a string value. Press Enter for the default (lesmi). +user> guest + +Option port. +Samba port number. +Enter a signed integer. Press Enter for the default (445). +port> + +Option pass. +Samba password. +Choose an alternative below. Press Enter for the default (n). +y) Yes, type in my own password +g) Generate random password +n) No, leave this optional password blank (default) +y/g/n> g +Password strength in bits. +64 is just about memorable +128 is secure +1024 is the maximum +Bits> 64 +Your password is: XXXX +Use this password? Please note that an obscured version of this +password (and not the password itself) will be stored under your +configuration file, so keep this generated password in a safe place. +y) Yes (default) +n) No +y/n> y + +Option domain. +Domain name for NTLM authentication. +Enter a string value. Press Enter for the default (WORKGROUP). +domain> + +Edit advanced config? +y) Yes +n) No (default) +y/n> n + +Configuration complete. +Options: +- type: samba +- host: localhost +- user: guest +- pass: *** ENCRYPTED *** +Keep this "remote" remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> d +``` + +{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/smb/smb.go then run make backenddocs" >}} +### Standard options + +Here are the Standard options specific to smb (SMB / CIFS). + +#### --smb-host + +SMB server hostname to connect to. + +E.g. "example.com". + +Properties: + +- Config: host +- Env Var: RCLONE_SMB_HOST +- Type: string +- Required: true + +#### --smb-user + +SMB username. + +Properties: + +- Config: user +- Env Var: RCLONE_SMB_USER +- Type: string +- Default: "$USER" + +#### --smb-port + +SMB port number. + +Properties: + +- Config: port +- Env Var: RCLONE_SMB_PORT +- Type: int +- Default: 445 + +#### --smb-pass + +SMB password. + +**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/). + +Properties: + +- Config: pass +- Env Var: RCLONE_SMB_PASS +- Type: string +- Required: false + +#### --smb-domain + +Domain name for NTLM authentication. + +Properties: + +- Config: domain +- Env Var: RCLONE_SMB_DOMAIN +- Type: string +- Default: "WORKGROUP" + +### Advanced options + +Here are the Advanced options specific to smb (SMB / CIFS). + +#### --smb-idle-timeout + +Max time before closing idle connections. + +If no connections have been returned to the connection pool in the time +given, rclone will empty the connection pool. + +Set to 0 to keep connections indefinitely. + + +Properties: + +- Config: idle_timeout +- Env Var: RCLONE_SMB_IDLE_TIMEOUT +- Type: Duration +- Default: 1m0s + +#### --smb-hide-special-share + +Hide special shares (e.g. print$) which users aren't supposed to access. + +Properties: + +- Config: hide_special_share +- Env Var: RCLONE_SMB_HIDE_SPECIAL_SHARE +- Type: bool +- Default: true + +#### --smb-case-insensitive + +Whether the server is configured to be case-insensitive. + +Always true on Windows shares. + +Properties: + +- Config: case_insensitive +- Env Var: RCLONE_SMB_CASE_INSENSITIVE +- Type: bool +- Default: true + +#### --smb-encoding + +The encoding for the backend. + +See the [encoding section in the overview](/overview/#encoding) for more info. + +Properties: + +- Config: encoding +- Env Var: RCLONE_SMB_ENCODING +- Type: MultiEncoder +- Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot + +{{< rem autogenerated options stop >}} diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 591219cf3..1499511ea 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -92,6 +92,7 @@ Seafile SFTP Sia + SMB / CIFS Storj SugarSync Uptobox diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 23cdee6db..b946a3c91 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -369,6 +369,9 @@ backends: - backend: "sia" remote: "TestSia:" fastlist: false + - backend: "smb" + remote: "TestSMB:rclone" + fastlist: false - backend: "storj" remote: "TestStorj:" fastlist: true diff --git a/fstest/testserver/init.d/TestSMB b/fstest/testserver/init.d/TestSMB new file mode 100755 index 000000000..788b5a84a --- /dev/null +++ b/fstest/testserver/init.d/TestSMB @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +NAME=smb +USER=rclone +PASS=GNF3Cqeu +WORKGROUP=thepub + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME dperson/samba \ + -p \ + -u "rclone;${PASS}" \ + -w "${WORKGROUP}" \ + -s "public;/share" \ + -s "rclone;/rclone;yes;no;no;rclone" + + echo type=smb + echo host=$(docker_ip) + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo domain=$WORKGROUP + echo _connect=$(docker_ip):139 +} + +. $(dirname "$0")/run.bash diff --git a/go.mod b/go.mod index 6358a144c..f40f1f11e 100644 --- a/go.mod +++ b/go.mod @@ -84,6 +84,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gdamore/encoding v1.0.0 // indirect + github.com/geoffgarside/ber v1.1.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -92,6 +93,7 @@ require ( github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hirochachacha/go-smb2 v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 544544307..26c74ec78 100644 --- a/go.sum +++ b/go.sum @@ -204,6 +204,8 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.5.2 h1:tKzG29kO9p2V++3oBY2W9zUjYu7IK1MENFeY/BzJSVY= github.com/gdamore/tcell/v2 v2.5.2/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo= +github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= +github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -353,6 +355,8 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= +github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -676,6 +680,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=