From c3af0a1ecaa2c43ba4e1bd8ee7ac6b5b6d1cd93d Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Sat, 11 May 2019 22:21:37 +0100
Subject: [PATCH] local: only calculate the required hashes for big speedup
Before this change we calculated all possible hashes for the file when
the `Hashes` method was called.
After we only calculate the Hash requested.
Almost all uses of `Hash` just need one checksum. This will slow down
`rclone lsjson` with the `--hash` flag. Perhaps lsjson should have a
`--hash-type` flag.
However it will speed up sync/copy/move/check/md5sum/sha1sum etc.
Before it took 12.4 seconds to md5sum a 1GB file, after it takes 3.1
seconds which is the same time the md5sum utility takes.
---
backend/local/local.go | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/backend/local/local.go b/backend/local/local.go
index 4506a3118..db09e7d7c 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -709,9 +709,10 @@ func (o *Object) Hash(r hash.Type) (string, error) {
o.fs.objectHashesMu.Lock()
hashes := o.hashes
+ hashValue, hashFound := o.hashes[r]
o.fs.objectHashesMu.Unlock()
- if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
+ if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil || !hashFound {
var in io.ReadCloser
if !o.translatedLink {
@@ -722,7 +723,7 @@ func (o *Object) Hash(r hash.Type) (string, error) {
if err != nil {
return "", errors.Wrap(err, "hash: failed to open")
}
- hashes, err = hash.Stream(in)
+ hashes, err = hash.StreamTypes(in, hash.NewHashSet(r))
closeErr := in.Close()
if err != nil {
return "", errors.Wrap(err, "hash: failed to read")
@@ -730,11 +731,16 @@ func (o *Object) Hash(r hash.Type) (string, error) {
if closeErr != nil {
return "", errors.Wrap(closeErr, "hash: failed to close")
}
+ hashValue = hashes[r]
o.fs.objectHashesMu.Lock()
- o.hashes = hashes
+ if o.hashes == nil {
+ o.hashes = hashes
+ } else {
+ o.hashes[r] = hashValue
+ }
o.fs.objectHashesMu.Unlock()
}
- return hashes[r], nil
+ return hashValue, nil
}
// Size returns the size of an object in bytes