github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-25 09:41:44 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Detect Fs from path to make operations consistent everywhere

Browse Source
This commit is contained in:
Nick Craig-Wood 2012-12-29 11:35:41 +00:00
parent 4028a4192f
commit c15ae179ee
5 changed files with 196 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
fs.go
View File

@ -37,6 +37,16 @@ type FsObjectsChan chan FsObject
type FsObjects []FsObject
// NewFs makes a new Fs object from the path
//
// FIXME make more generic in future
func NewFs(path string) (Fs, error) {
if swiftMatch.MatchString(path) {
return NewFsSwift(path)
}
return NewFsLocal(path)
}
// checkClose is a utility function used to check the return from
// Close in a defer statement.
func checkClose(c io.Closer, err *error) {

7
fs_local.go
View File

@ -26,6 +26,13 @@ type FsObjectLocal struct {
// ------------------------------------------------------------
// NewFsLocal contstructs an FsLocal from the path
func NewFsLocal(root string) (*FsLocal, error) {
root = path.Clean(root)
f := &FsLocal{root: root}
return f, nil
}
// Return an FsObject from a path
//
// May return nil if an error occurred

83
fs_swift.go
View File

@ -2,10 +2,14 @@
package main
import (
"errors"
"flag"
"fmt"
"github.com/ncw/swift"
"io"
"log"
"os"
"regexp"
"strings"
"time"
)
@ -28,6 +32,85 @@ type FsObjectSwift struct {
// ------------------------------------------------------------
// Globals
var (
// Flags
// FIXME make these part of swift so we get a standard set of flags?
authUrl = flag.String("auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
)
// Pattern to match a swift url
var swiftMatch = regexp.MustCompile(`^([^/:]+):(.*)$`)
// parseParse parses a swift 'url'
func parsePath(path string) (container, directory string, err error) {
parts := swiftMatch.FindAllStringSubmatch(path, -1)
if len(parts) != 1 || len(parts[0]) != 3 {
err = fmt.Errorf("Couldn't parse swift url %q", path)
} else {
container, directory = parts[0][1], parts[0][2]
directory = strings.Trim(directory, "/")
}
return
}
// swiftConnection makes a connection to swift
func swiftConnection() (*swift.Connection, error) {
if *userName == "" {
return nil, errors.New("Need -user or environmental variable ST_USER")
}
if *apiKey == "" {
return nil, errors.New("Need -key or environmental variable ST_KEY")
}
if *authUrl == "" {
return nil, errors.New("Need -auth or environmental variable ST_AUTH")
}
c := &swift.Connection{
UserName: *userName,
ApiKey: *apiKey,
AuthUrl: *authUrl,
}
err := c.Authenticate()
if err != nil {
return nil, err
}
return c, nil
}
// NewFsSwift contstructs an FsSwift from the path, container:path
func NewFsSwift(path string) (*FsSwift, error) {
container, directory, err := parsePath(path)
if err != nil {
return nil, err
}
if directory != "" {
return nil, fmt.Errorf("Directories not supported yet in %q", path)
}
c, err := swiftConnection()
if err != nil {
return nil, err
}
f := &FsSwift{c: *c, container: container}
return f, nil
}
// Lists the containers
func SwiftContainers() {
c, err := swiftConnection()
if err != nil {
log.Fatalf("Couldn't connect: %s", err)
}
containers, err := c.ContainersAll(nil)
if err != nil {
log.Fatalf("Couldn't list containers: %s", err)
}
for _, container := range containers {
fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name)
}
}
// Return an FsObject from a path
//
// May return nil if an error occurred

65
notes.txt
View File

@ -1,40 +1,24 @@
Todo
* Add sync command (like rsync with delete)
* Check logging in various parts
* Make logging controllable with flags
* progress meter would be nice! Do this by wrapping the Reader with a progress bar
* Do bandwidth limit by wrapping the Reader too
* Maybe using https://jra-go.googlecode.com/hg/linkio/ which will work for multiple
uploads or downloads.
* code.google.com/p/mxk/go1/flowcontrol - only does one flow at once
* Or maybe put into swift library.
* Make swift timeouts be settable with command line parameters
* Check the locking in swift module!
* Windows paths? Do we need to translate / and \?
* Make a fs.Errorf and count errors and log them at a different level
Ideas
* remote copy container to another container
* check local is same as remote
* syncup/syncdown (like rsync with delete)
* use container: /path/syntax like rsync?
* optimise remote copy container to another container using remote
copy if local is same as remote
* Allow subpaths container:/sub/path
* allow local/local too
* progress reports
* stats
* Add bandwidth limit?
make 100% compatible with swift.py?
Make Env vars compatible with st?
Get and put the metadata in the libray (x-object-meta-mtime) when getting and putting a file?
This also puts meta-mtime
https://github.com/gholt/swiftly
As an integer, but it does parse it as a float
subargs.append('x-object-meta-mtime:%d' % getmtime(options.input_))
Need an iterate all objects routine... Could use a channel
- could just be a flag to Objects()
FIXME progress meter would be nice! Do this by wrapping the Reader with a progress bar
Do bandwidth limit by wrapping the Reader too
Maybe using https://jra-go.googlecode.com/hg/linkio/ which will work for multiple
uploads or downloads.
* code.google.com/p/mxk/go1/flowcontrol - only does one flow at once
Or maybe put into swift library.
Windows paths? Do we need to translate / and \?
Make swift timeouts be settable with command line parameters
* look at auth from env in s3 module - add to swift?
Make a wrapper in connection which
* measures bandwidth and reports it
@ -43,10 +27,17 @@ Make a wrapper in connection which
* does timeouts by setting a limit, seeing whether io has happened
and resetting it if it has
Check the locking in swift module!
Need to make directory objects otherwise can't upload an empty directory
* Or could upload empty directories only?
* Can't purge a local filesystem because it leaves the directories behind
Make a fs.Errorf and count errors and log them at a different level
s3
--
Can maybe set last modified?
https://forums.aws.amazon.com/message.jspa?messageID=214062
Otherwise can set metadata
Returns etag and last modified in bucket list

202
swiftsync.go
View File

@ -6,7 +6,6 @@ package main
import (
"flag"
"fmt"
"github.com/ncw/swift"
"log"
"os"
"runtime"
@ -22,12 +21,8 @@ var (
snet = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
verbose = flag.Bool("verbose", false, "Print lots more stuff")
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
// FIXME make these part of swift so we get a standard set of flags?
authUrl = flag.String("auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
transfers = flag.Int("transfers", 4, "Number of file transfers to run in parallel.")
checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
transfers = flag.Int("transfers", 4, "Number of file transfers to run in parallel.")
)
// Read FsObjects~s on in send to out if they need uploading
@ -65,7 +60,7 @@ func Copier(in FsObjectsChan, fdst Fs, wg *sync.WaitGroup) {
}
// Copies fsrc into fdst
func Copy(fsrc, fdst Fs) {
func Copy(fdst, fsrc Fs) {
err := fdst.Mkdir()
if err != nil {
log.Fatal("Failed to make destination")
@ -93,40 +88,9 @@ func Copy(fsrc, fdst Fs) {
copierWg.Wait()
}
// Syncs a directory into a container
func upload(c *swift.Connection, args []string) {
root, container := args[0], args[1]
// FIXME
fsrc := &FsLocal{root: root}
fdst := &FsSwift{c: *c, container: container}
Copy(fsrc, fdst)
}
// Syncs a container into a directory
//
// FIXME need optional stat in FsObject and to be able to make FsObjects from ObjectsAll
//
// FIXME should download and stat many at once
func download(c *swift.Connection, args []string) {
container, root := args[0], args[1]
// FIXME
fsrc := &FsSwift{c: *c, container: container}
fdst := &FsLocal{root: root}
Copy(fsrc, fdst)
}
// Lists the containers
func listContainers(c *swift.Connection) {
containers, err := c.ContainersAll(nil)
if err != nil {
log.Fatalf("Couldn't list containers: %s", err)
}
for _, container := range containers {
fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name)
}
// Copy~s from source to dest
func copy_(fdst, fsrc Fs) {
Copy(fdst, fsrc)
}
// List the Fs to stdout
@ -147,57 +111,34 @@ func List(f Fs) {
}
// Lists files in a container
func list(c *swift.Connection, args []string) {
if len(args) == 0 {
listContainers(c)
func list(fdst, fsrc Fs) {
if fdst == nil {
SwiftContainers()
return
}
container := args[0]
// FIXME
f := &FsSwift{c: *c, container: container}
List(f)
List(fdst)
}
// Local lists files
func llist(c *swift.Connection, args []string) {
root := args[0]
// FIXME
f := &FsLocal{root: root}
List(f)
}
// Makes a container
func mkdir(c *swift.Connection, args []string) {
container := args[0]
// FIXME
fdst := &FsSwift{c: *c, container: container}
// Makes a destination directory or container
func mkdir(fdst, fsrc Fs) {
err := fdst.Mkdir()
if err != nil {
log.Fatalf("Couldn't create container %q: %s", container, err)
log.Fatalf("Mkdir failed: %s", err)
}
}
// Removes a container
func rmdir(c *swift.Connection, args []string) {
container := args[0]
// FIXME
fdst := &FsSwift{c: *c, container: container}
// Removes a container but not if not empty
func rmdir(fdst, fsrc Fs) {
err := fdst.Rmdir()
if err != nil {
log.Fatalf("Couldn't delete container %q: %s", container, err)
log.Fatalf("Rmdir failed: %s", err)
}
}
// Removes a container and all of its contents
//
// FIXME should make FsObjects and use debugging
func purge(c *swift.Connection, args []string) {
container := args[0]
// FIXME
fdst := &FsSwift{c: *c, container: container}
// FIXME doesn't delete local directories
func purge(fdst, fsrc Fs) {
to_be_deleted := fdst.List()
var wg sync.WaitGroup
@ -219,14 +160,14 @@ func purge(c *swift.Connection, args []string) {
log.Printf("Waiting for deletions to finish")
wg.Wait()
log.Printf("Deleting container")
rmdir(c, args)
log.Printf("Deleting path")
rmdir(fdst, fsrc)
}
type Command struct {
name string
help string
run func(*swift.Connection, []string)
run func(fdst, fsrc Fs)
minArgs, maxArgs int
}
@ -245,62 +186,55 @@ func (cmd *Command) checkArgs(args []string) {
var Commands = []Command{
{
"upload",
`<directory> <container>
Upload the local directory to the remote container. Doesn't
upload unchanged files, testing first by modification time
then by MD5SUM
"copy",
`<source> <destination>
Copy the source to the destination. Doesn't transfer
unchanged files, testing first by modification time then by
MD5SUM. Doesn't delete files from the destination.
`,
upload,
2, 2,
},
{
"download",
`<container> <directory>
Download the container to the local directory. Doesn't
download unchanged files
`,
download,
copy_,
2, 2,
},
{
"ls",
`[<container>]
List the containers if no parameter supplied or the contents
of <container>
`[<path>]
List the path. If no parameter is supplied then it lists the
available swift containers.
`,
list,
0, 1,
},
{
"lls",
`[<directory>]
List the directory
`,
llist,
1, 1,
},
{
"mkdir",
`<container>
Make the container if it doesn't already exist
`<path>
Make the path if it doesn't already exist
`,
mkdir,
1, 1,
},
{
"rmdir",
`<container>
Remove the container. Note that you can't remove a container with
objects in - use rm for that
`<path>
Remove the path. Note that you can't remove a path with
objects in it, use purge for that
`,
rmdir,
1, 1,
},
{
"purge",
`<container>
Remove the container and all of the contents.
`<path>
Remove the path and all of its contents.
`,
purge,
1, 1,
@ -314,6 +248,7 @@ func syntaxError() {
Syntax: [options] subcommand <parameters> <parameters...>
Subcommands:
`)
for i := range Commands {
cmd := &Commands[i]
@ -350,30 +285,10 @@ func main() {
defer pprof.StopCPUProfile()
}
fmt.Println(args)
if len(args) < 1 {
fatal("No command supplied\n")
}
if *userName == "" {
log.Fatal("Need -user or environmental variable ST_USER")
}
if *apiKey == "" {
log.Fatal("Need -key or environmental variable ST_KEY")
}
if *authUrl == "" {
log.Fatal("Need -auth or environmental variable ST_AUTH")
}
c := &swift.Connection{
UserName: *userName,
ApiKey: *apiKey,
AuthUrl: *authUrl,
}
err := c.Authenticate()
if err != nil {
log.Fatal("Failed to authenticate", err)
}
cmd := strings.ToLower(args[0])
args = args[1:]
@ -396,5 +311,24 @@ func main() {
log.Fatalf("Unknown command %q", cmd)
}
found.checkArgs(args)
found.run(c, args)
// Make source and destination fs
var fdst, fsrc Fs
var err error
if len(args) >= 1 {
fdst, err = NewFs(args[0])
if err != nil {
log.Fatal("Failed to create file system: ", err)
}
}
if len(args) >= 2 {
fsrc, err = NewFs(args[1])
if err != nil {
log.Fatal("Failed to create destination file system: ", err)
}
fsrc, fdst = fdst, fsrc
}
// Run the actual command
found.run(fdst, fsrc)
}