From 90a2c86eb36d318be1bb1d08ea6817ade3d9484a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 31 Dec 2012 17:31:19 +0000 Subject: [PATCH] Implement check and help commands --- fs.go | 36 +++++++++------ notes.txt | 2 +- swiftsync.go | 122 ++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 129 insertions(+), 31 deletions(-) diff --git a/fs.go b/fs.go index cdc33fb8e..c123c2cd7 100644 --- a/fs.go +++ b/fs.go @@ -74,6 +74,27 @@ func checkClose(c io.Closer, err *error) { } } +// Check the two files to see if the MD5sums are the same +// +// May return an error which will already have been logged +// +// If an error is returned it will return false +func CheckMd5sums(src, dst FsObject) (bool, error) { + srcMd5, err := src.Md5sum() + if err != nil { + FsLog(src, "Failed to calculate src md5: %s", err) + return false, err + } + dstMd5, err := dst.Md5sum() + if err != nil { + FsLog(dst, "Failed to calculate dst md5: %s", err) + return false, err + } + // FsDebug("Src MD5 %s", srcMd5) + // FsDebug("Dst MD5 %s", obj.Hash) + return srcMd5 == dstMd5, nil +} + // Checks to see if the src and dst objects are equal by looking at // size, mtime and MD5SUM // @@ -114,19 +135,8 @@ func Equal(src, dst FsObject) bool { // mtime is unreadable or different but size is the same so // check the MD5SUM - srcMd5, err := src.Md5sum() - if err != nil { - FsDebug(src, "Failed to calculate src md5: %s", err) - return false - } - dstMd5, err := dst.Md5sum() - if err != nil { - FsDebug(dst, "Failed to calculate dst md5: %s", err) - return false - } - // FsDebug("Src MD5 %s", srcMd5) - // FsDebug("Dst MD5 %s", obj.Hash) - if srcMd5 != dstMd5 { + same, err := CheckMd5sums(src, dst) + if !same { FsDebug(src, "Md5sums differ") return false } diff --git a/notes.txt b/notes.txt index c93cd0adc..2e36f3ef0 100644 --- a/notes.txt +++ b/notes.txt @@ -1,4 +1,5 @@ Todo + * FIXME: More -dry-run checks for object transfer * Check logging in various parts * Make logging controllable with flags (mostly done) * progress meter would be nice! Do this by wrapping the Reader with a progress bar @@ -12,7 +13,6 @@ Todo * Windows paths? Do we need to translate / and \? * Make a fs.Errorf and count errors and log them at a different level * add -modify-window flag - fs should keep knowledge of resolution - * add check command to compare local MD5SUMs with remote Ideas * optimise remote copy container to another container using remote diff --git a/swiftsync.go b/swiftsync.go index 3da76f94e..b2acd14eb 100644 --- a/swiftsync.go +++ b/swiftsync.go @@ -125,16 +125,16 @@ func Sync(fdst, fsrc Fs) { // Read the destination files first // FIXME could do this in parallel and make it use less memory delFiles := make(map[string]FsObject) - for dstFile := range fdst.List() { - delFiles[dstFile.Remote()] = dstFile + for dst := range fdst.List() { + delFiles[dst.Remote()] = dst } // Read source files checking them off against dest files to_be_checked := make(FsObjectsChan, *transfers) go func() { - for srcFile := range fsrc.List() { - delete(delFiles, srcFile.Remote()) - to_be_checked <- srcFile + for src := range fsrc.List() { + delete(delFiles, src.Remote()) + to_be_checked <- src } close(to_be_checked) }() @@ -172,6 +172,74 @@ func Sync(fdst, fsrc Fs) { DeleteFiles(toDelete) } +// Checks the files in fsrc and fdst according to Size and MD5SUM +func Check(fdst, fsrc Fs) { + // Read the destination files first + // FIXME could do this in parallel and make it use less memory + dstFiles := make(map[string]FsObject) + for dst := range fdst.List() { + dstFiles[dst.Remote()] = dst + } + + // Read the source files checking them against dstFiles + // FIXME could do this in parallel and make it use less memory + srcFiles := make(map[string]FsObject) + commonFiles := make(map[string][]FsObject) + for src := range fsrc.List() { + remote := src.Remote() + if dst, ok := dstFiles[remote]; ok { + commonFiles[remote] = []FsObject{dst, src} + delete(dstFiles, remote) + } else { + srcFiles[remote] = src + } + } + + log.Printf("Files in %s but not in %s", fdst, fsrc) + for remote := range dstFiles { + log.Printf(remote) + } + + log.Printf("Files in %s but not in %s", fsrc, fdst) + for remote := range srcFiles { + log.Printf(remote) + } + + checks := make(chan []FsObject, *transfers) + go func() { + for _, check := range commonFiles { + checks <- check + } + close(checks) + }() + + var checkerWg sync.WaitGroup + checkerWg.Add(*checkers) + for i := 0; i < *checkers; i++ { + go func() { + defer checkerWg.Done() + for check := range checks { + dst, src := check[0], check[1] + if src.Size() != dst.Size() { + FsLog(src, "Sizes differ") + continue + } + same, err := CheckMd5sums(src, dst) + if err != nil { + continue + } + if !same { + FsLog(src, "Md5sums differ") + } + FsDebug(src, "OK") + } + }() + } + + log.Printf("Waiting for checks to finish") + checkerWg.Wait() +} + // List the Fs to stdout func List(f Fs) { // FIXME error? @@ -268,9 +336,8 @@ var Commands = []Command{ unchanged files, testing first by modification time then by MD5SUM. Deletes any files that exist in source that don't exist in destination. Since this can cause data loss, test - first with the -dry-run flag. + first with the -dry-run flag.`, -`, Sync, 2, 2, }, @@ -279,9 +346,8 @@ var Commands = []Command{ `[] List the path. If no parameter is supplied then it lists the - available swift containers. + available swift containers.`, -`, list, 0, 1, }, @@ -289,9 +355,8 @@ var Commands = []Command{ "mkdir", ` - Make the path if it doesn't already exist + Make the path if it doesn't already exist`, -`, mkdir, 1, 1, }, @@ -300,9 +365,8 @@ var Commands = []Command{ ` Remove the path. Note that you can't remove a path with - objects in it, use purge for that + objects in it, use purge for that.`, -`, rmdir, 1, 1, }, @@ -310,12 +374,31 @@ var Commands = []Command{ "purge", ` - Remove the path and all of its contents. + Remove the path and all of its contents.`, -`, purge, 1, 1, }, + { + "check", + ` + + Checks the files in the source and destination match. It + compares sizes and MD5SUMs and prints a report of files which + don't match. It doesn't alter the source or destination.`, + + Check, + 2, 2, + }, + { + "help", + ` + + This help.`, + + nil, + 0, 0, + }, } // syntaxError prints the syntax @@ -329,7 +412,7 @@ Subcommands: `) for i := range Commands { cmd := &Commands[i] - fmt.Fprintf(os.Stderr, " %s: %s\n", cmd.name, cmd.help) + fmt.Fprintf(os.Stderr, " %s: %s\n\n", cmd.name, cmd.help) } fmt.Fprintf(os.Stderr, "Options:\n") @@ -407,5 +490,10 @@ func main() { } // Run the actual command - found.run(fdst, fsrc) + if found.run != nil { + found.run(fdst, fsrc) + } else { + syntaxError() + } + }