fs: make ConfigString properly reverse suffixed file systems

Before this change we renamed file systems with overridden config with
{suffix}.

However this meant that ConfigString produced a value which wouldn't
re-create the file system.

This uses an internal hash to keep note of what config goes which
which {suffix} in order to remake the config properly.
This commit is contained in:
Nick Craig-Wood 2023-04-28 11:58:49 +01:00
parent 6b670bd439
commit 3567a47258
2 changed files with 73 additions and 5 deletions

View File

@ -9,11 +9,18 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/fspath"
) )
// Store the hashes of the overridden config
var (
overriddenConfigMu sync.Mutex
overriddenConfig = make(map[string]string)
)
// NewFs makes a new Fs object from the path // NewFs makes a new Fs object from the path
// //
// The path is of the form remote:path // The path is of the form remote:path
@ -37,18 +44,25 @@ func NewFs(ctx context.Context, path string) (Fs, error) {
extraConfig := overridden.String() extraConfig := overridden.String()
//Debugf(nil, "detected overridden config %q", extraConfig) //Debugf(nil, "detected overridden config %q", extraConfig)
md5sumBinary := md5.Sum([]byte(extraConfig)) md5sumBinary := md5.Sum([]byte(extraConfig))
suffix := base64.RawURLEncoding.EncodeToString(md5sumBinary[:]) configHash := base64.RawURLEncoding.EncodeToString(md5sumBinary[:])
// 5 characters length is 5*6 = 30 bits of base64 // 5 characters length is 5*6 = 30 bits of base64
const maxLength = 5 overriddenConfigMu.Lock()
if len(suffix) > maxLength { var suffix string
suffix = suffix[:maxLength] for maxLength := 5; ; maxLength++ {
suffix = "{" + configHash[:maxLength] + "}"
existingExtraConfig, ok := overriddenConfig[suffix]
if !ok || existingExtraConfig == extraConfig {
break
}
} }
suffix = "{" + suffix + "}"
Debugf(configName, "detected overridden config - adding %q suffix to name", suffix) Debugf(configName, "detected overridden config - adding %q suffix to name", suffix)
// Add the suffix to the config name // Add the suffix to the config name
// //
// These need to work as filesystem names as the VFS cache will use them // These need to work as filesystem names as the VFS cache will use them
configName += suffix configName += suffix
// Store the config suffixes for reversing in ConfigString
overriddenConfig[suffix] = extraConfig
overriddenConfigMu.Unlock()
} }
f, err := fsInfo.NewFs(ctx, configName, fsPath, config) f, err := fsInfo.NewFs(ctx, configName, fsPath, config)
if f != nil && (err == nil || err == ErrorIsFile) { if f != nil && (err == nil || err == ErrorIsFile) {
@ -105,6 +119,17 @@ func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, conne
// to configure the Fs as passed to fs.NewFs // to configure the Fs as passed to fs.NewFs
func ConfigString(f Fs) string { func ConfigString(f Fs) string {
name := f.Name() name := f.Name()
if open := strings.IndexRune(name, '{'); open >= 0 && strings.HasSuffix(name, "}") {
suffix := name[open:]
overriddenConfigMu.Lock()
config, ok := overriddenConfig[suffix]
overriddenConfigMu.Unlock()
if ok {
name = name[:open] + "," + config
} else {
Errorf(f, "Failed to find config for suffix %q", suffix)
}
}
root := f.Root() root := f.Root()
if name == "local" && f.Features().IsLocal { if name == "local" && f.Features().IsLocal {
return root return root

43
fs/newfs_test.go Normal file
View File

@ -0,0 +1,43 @@
package fs_test
import (
"context"
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest/mockfs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewFs(t *testing.T) {
ctx := context.Background()
// Register mockfs temporarily
oldRegistry := fs.Registry
mockfs.Register()
defer func() {
fs.Registry = oldRegistry
}()
f1, err := fs.NewFs(ctx, ":mockfs:/tmp")
require.NoError(t, err)
assert.Equal(t, ":mockfs", f1.Name())
assert.Equal(t, "/tmp", f1.Root())
assert.Equal(t, ":mockfs:/tmp", fs.ConfigString(f1))
f2, err := fs.NewFs(ctx, ":mockfs,potato:/tmp")
require.NoError(t, err)
assert.Equal(t, ":mockfs{S_NHG}", f2.Name())
assert.Equal(t, "/tmp", f2.Root())
assert.Equal(t, ":mockfs,potato='true':/tmp", fs.ConfigString(f2))
f3, err := fs.NewFs(ctx, ":mockfs,potato='true':/tmp")
require.NoError(t, err)
assert.Equal(t, ":mockfs{S_NHG}", f3.Name())
assert.Equal(t, "/tmp", f3.Root())
assert.Equal(t, ":mockfs,potato='true':/tmp", fs.ConfigString(f3))
}