2015-09-23 01:47:16 +08:00
// Package fstest provides utilities for testing the Fs
2014-07-25 05:50:11 +08:00
package fstest
// FIXME put name of test FS in Fs structure
import (
2016-01-17 18:08:28 +08:00
"bytes"
2024-03-01 08:49:34 +08:00
"compress/gzip"
2019-06-17 16:34:30 +08:00
"context"
2016-07-11 18:36:46 +08:00
"flag"
2016-06-30 00:59:31 +08:00
"fmt"
2016-01-17 18:08:28 +08:00
"io"
2014-07-25 05:50:11 +08:00
"os"
2017-08-09 22:51:27 +08:00
"path"
2015-02-07 23:52:06 +08:00
"path/filepath"
2016-01-24 20:37:46 +08:00
"regexp"
2017-09-17 21:05:33 +08:00
"runtime"
2016-11-26 05:52:43 +08:00
"sort"
2014-07-25 05:50:11 +08:00
"strings"
2014-08-02 00:58:39 +08:00
"testing"
2014-07-25 05:50:11 +08:00
"time"
2019-07-29 01:47:38 +08:00
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/config"
2021-03-10 23:40:34 +08:00
"github.com/rclone/rclone/fs/config/configfile"
2019-07-29 01:47:38 +08:00
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/walk"
2024-04-24 18:38:22 +08:00
"github.com/rclone/rclone/fstest/testy"
2019-08-06 19:44:08 +08:00
"github.com/rclone/rclone/lib/random"
2016-06-30 00:59:31 +08:00
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
2017-09-17 21:05:33 +08:00
"golang.org/x/text/unicode/norm"
2014-07-25 05:50:11 +08:00
)
2017-07-25 05:46:43 +08:00
// Globals
2016-01-24 20:37:46 +08:00
var (
2017-07-25 05:46:43 +08:00
RemoteName = flag . String ( "remote" , "" , "Remote to test with, defaults to local filesystem" )
Verbose = flag . Bool ( "verbose" , false , "Set to enable logging" )
DumpHeaders = flag . Bool ( "dump-headers" , false , "Set to dump headers (needs -verbose)" )
DumpBodies = flag . Bool ( "dump-bodies" , false , "Set to dump bodies (needs -verbose)" )
Individual = flag . Bool ( "individual" , false , "Make individual bucket/container/directory for each test - much slower" )
LowLevelRetries = flag . Int ( "low-level-retries" , 10 , "Number of low level retries" )
UseListR = flag . Bool ( "fast-list" , false , "Use recursive list if available. Uses more memory but fewer transactions." )
2019-09-17 01:52:41 +08:00
// SizeLimit signals tests to skip maximum test file size and skip inappropriate runs
SizeLimit = flag . Int64 ( "size-limit" , 0 , "Limit maximum test file size" )
2017-02-22 18:14:40 +08:00
// ListRetries is the number of times to retry a listing to overcome eventual consistency
2020-05-08 03:28:41 +08:00
ListRetries = flag . Int ( "list-retries" , 3 , "Number or times to retry listing" )
2017-07-25 05:46:43 +08:00
// MatchTestRemote matches the remote names used for testing
2024-04-23 07:34:20 +08:00
MatchTestRemote = regexp . MustCompile ( ` ^rclone-test-[abcdefghijklmnopqrstuvwxyz0123456789] { 12}$ ` )
2016-01-24 20:37:46 +08:00
)
2017-07-25 05:46:43 +08:00
// Initialise rclone for testing
func Initialise ( ) {
2020-11-06 00:59:59 +08:00
ctx := context . Background ( )
2020-11-05 19:33:32 +08:00
ci := fs . GetConfig ( ctx )
2017-07-25 05:46:43 +08:00
// Never ask for passwords, fail instead.
// If your local config is encrypted set environment variable
// "RCLONE_CONFIG_PASS=hunter2" (or your password)
2020-11-05 19:33:32 +08:00
ci . AskPassword = false
2018-01-23 19:25:18 +08:00
// Override the config file from the environment - we don't
// parse the flags any more so this doesn't happen
// automatically
if envConfig := os . Getenv ( "RCLONE_CONFIG" ) ; envConfig != "" {
2021-04-08 23:49:47 +08:00
_ = config . SetConfigPath ( envConfig )
2018-01-23 19:25:18 +08:00
}
2021-04-27 05:37:49 +08:00
configfile . Install ( )
2021-03-11 04:16:17 +08:00
accounting . Start ( ctx )
2017-07-25 05:46:43 +08:00
if * Verbose {
2020-11-05 19:33:32 +08:00
ci . LogLevel = fs . LogLevelDebug
2017-07-25 05:46:43 +08:00
}
2017-11-21 04:21:44 +08:00
if * DumpHeaders {
2020-11-05 19:33:32 +08:00
ci . Dump |= fs . DumpHeaders
2017-11-21 04:21:44 +08:00
}
if * DumpBodies {
2020-11-05 19:33:32 +08:00
ci . Dump |= fs . DumpBodies
2017-11-21 04:21:44 +08:00
}
2020-11-05 19:33:32 +08:00
ci . LowLevelRetries = * LowLevelRetries
ci . UseListR = * UseListR
2017-07-25 05:46:43 +08:00
}
2015-09-23 01:47:16 +08:00
// Item represents an item for checking
2014-07-25 05:50:11 +08:00
type Item struct {
Path string
2018-01-13 00:30:54 +08:00
Hashes map [ hash . Type ] string
2014-07-25 05:50:11 +08:00
ModTime time . Time
Size int64
}
2016-01-17 18:08:28 +08:00
// NewItem creates an item from a string content
func NewItem ( Path , Content string , modTime time . Time ) Item {
i := Item {
Path : Path ,
ModTime : modTime ,
Size : int64 ( len ( Content ) ) ,
}
2018-01-13 00:30:54 +08:00
hash := hash . NewMultiHasher ( )
2016-01-17 18:08:28 +08:00
buf := bytes . NewBufferString ( Content )
_ , err := io . Copy ( hash , buf )
if err != nil {
2024-08-18 22:58:35 +08:00
fs . Fatalf ( nil , "Failed to create item: %v" , err )
2016-01-17 18:08:28 +08:00
}
i . Hashes = hash . Sums ( )
return i
}
2015-09-23 01:47:16 +08:00
// CheckTimeEqualWithPrecision checks the times are equal within the
// precision, returns the delta and a flag
2015-08-17 06:24:34 +08:00
func CheckTimeEqualWithPrecision ( t0 , t1 time . Time , precision time . Duration ) ( time . Duration , bool ) {
dt := t0 . Sub ( t1 )
if dt >= precision || dt <= - precision {
return dt , false
}
return dt , true
}
2020-04-21 22:46:01 +08:00
// AssertTimeEqualWithPrecision checks that want is within precision
// of got, asserting that with t and logging remote
func AssertTimeEqualWithPrecision ( t * testing . T , remote string , want , got time . Time , precision time . Duration ) {
dt , ok := CheckTimeEqualWithPrecision ( want , got , precision )
assert . True ( t , ok , fmt . Sprintf ( "%s: Modification time difference too big |%s| > %s (want %s vs got %s) (precision %s)" , remote , dt , precision , want , got , precision ) )
}
2015-09-23 01:47:16 +08:00
// CheckModTime checks the mod time to the given precision
2014-08-02 00:58:39 +08:00
func ( i * Item ) CheckModTime ( t * testing . T , obj fs . Object , modTime time . Time , precision time . Duration ) {
2020-04-21 22:46:01 +08:00
AssertTimeEqualWithPrecision ( t , obj . Remote ( ) , i . ModTime , modTime , precision )
2014-07-27 00:18:29 +08:00
}
2016-01-17 19:27:31 +08:00
// CheckHashes checks all the hashes the object supports are correct
func ( i * Item ) CheckHashes ( t * testing . T , obj fs . Object ) {
2016-06-30 00:59:31 +08:00
require . NotNil ( t , obj )
2016-01-11 20:39:33 +08:00
types := obj . Fs ( ) . Hashes ( ) . Array ( )
2018-01-13 00:30:54 +08:00
for _ , Hash := range types {
2016-01-11 20:39:33 +08:00
// Check attributes
2019-06-17 16:34:30 +08:00
sum , err := obj . Hash ( context . Background ( ) , Hash )
2016-06-30 00:59:31 +08:00
require . NoError ( t , err )
2018-01-13 00:30:54 +08:00
assert . True ( t , hash . Equals ( i . Hashes [ Hash ] , sum ) , fmt . Sprintf ( "%s/%s: %v hash incorrect - expecting %q got %q" , obj . Fs ( ) . String ( ) , obj . Remote ( ) , Hash , i . Hashes [ Hash ] , sum ) )
2014-07-25 05:50:11 +08:00
}
2016-01-17 19:27:31 +08:00
}
// Check checks all the attributes of the object are correct
func ( i * Item ) Check ( t * testing . T , obj fs . Object , precision time . Duration ) {
i . CheckHashes ( t , obj )
2017-08-04 03:42:35 +08:00
assert . Equal ( t , i . Size , obj . Size ( ) , fmt . Sprintf ( "%s: size incorrect file=%d vs obj=%d" , i . Path , i . Size , obj . Size ( ) ) )
2019-06-17 16:34:30 +08:00
i . CheckModTime ( t , obj , obj . ModTime ( context . Background ( ) ) , precision )
2014-07-25 05:50:11 +08:00
}
2017-09-17 21:05:33 +08:00
// Normalize runs a utf8 normalization on the string if running on OS
// X. This is because OS X denormalizes file names it writes to the
// local file system.
func Normalize ( name string ) string {
if runtime . GOOS == "darwin" {
name = norm . NFC . String ( name )
}
return name
}
2015-09-23 01:47:16 +08:00
// Items represents all items for checking
2014-07-25 05:50:11 +08:00
type Items struct {
2015-09-11 17:37:12 +08:00
byName map [ string ] * Item
byNameAlt map [ string ] * Item
items [ ] Item
2014-07-25 05:50:11 +08:00
}
2015-09-23 01:47:16 +08:00
// NewItems makes an Items
2014-07-25 05:50:11 +08:00
func NewItems ( items [ ] Item ) * Items {
is := & Items {
2015-09-11 17:37:12 +08:00
byName : make ( map [ string ] * Item ) ,
byNameAlt : make ( map [ string ] * Item ) ,
items : items ,
2014-07-25 05:50:11 +08:00
}
// Fill up byName
for i := range items {
2017-09-17 21:05:33 +08:00
is . byName [ Normalize ( items [ i ] . Path ) ] = & items [ i ]
2014-07-25 05:50:11 +08:00
}
return is
}
2015-09-23 01:47:16 +08:00
// Find checks off an item
2014-08-02 00:58:39 +08:00
func ( is * Items ) Find ( t * testing . T , obj fs . Object , precision time . Duration ) {
2017-09-17 21:05:33 +08:00
remote := Normalize ( obj . Remote ( ) )
i , ok := is . byName [ remote ]
2014-07-25 05:50:11 +08:00
if ! ok {
2017-09-17 21:05:33 +08:00
i , ok = is . byNameAlt [ remote ]
assert . True ( t , ok , fmt . Sprintf ( "Unexpected file %q" , remote ) )
2014-07-25 05:50:11 +08:00
}
2016-07-11 18:36:46 +08:00
if i != nil {
delete ( is . byName , i . Path )
i . Check ( t , obj , precision )
}
2014-07-25 05:50:11 +08:00
}
2015-09-23 01:47:16 +08:00
// Done checks all finished
2014-08-02 00:58:39 +08:00
func ( is * Items ) Done ( t * testing . T ) {
2014-07-25 05:50:11 +08:00
if len ( is . byName ) != 0 {
for name := range is . byName {
2015-12-29 08:16:53 +08:00
t . Logf ( "Not found %q" , name )
2014-07-25 05:50:11 +08:00
}
}
2016-07-11 18:36:46 +08:00
assert . Equal ( t , 0 , len ( is . byName ) , fmt . Sprintf ( "%d objects not found" , len ( is . byName ) ) )
2014-07-25 05:50:11 +08:00
}
2017-01-20 01:25:57 +08:00
// makeListingFromItems returns a string representation of the items
2017-01-21 01:12:05 +08:00
//
// it returns two possible strings, one normal and one for windows
2018-11-02 20:16:13 +08:00
func makeListingFromItems ( items [ ] Item ) string {
nameLengths := make ( [ ] string , len ( items ) )
2017-01-20 01:25:57 +08:00
for i , item := range items {
2018-11-02 20:16:13 +08:00
remote := Normalize ( item . Path )
nameLengths [ i ] = fmt . Sprintf ( "%s (%d)" , remote , item . Size )
2017-01-20 01:25:57 +08:00
}
2018-11-02 20:16:13 +08:00
sort . Strings ( nameLengths )
return strings . Join ( nameLengths , ", " )
2017-01-20 01:25:57 +08:00
}
// makeListingFromObjects returns a string representation of the objects
func makeListingFromObjects ( objs [ ] fs . Object ) string {
nameLengths := make ( [ ] string , len ( objs ) )
for i , obj := range objs {
2017-09-17 21:05:33 +08:00
nameLengths [ i ] = fmt . Sprintf ( "%s (%d)" , Normalize ( obj . Remote ( ) ) , obj . Size ( ) )
2017-01-20 01:25:57 +08:00
}
sort . Strings ( nameLengths )
return strings . Join ( nameLengths , ", " )
}
2017-08-09 22:51:27 +08:00
// filterEmptyDirs removes any empty (or containing only directories)
// directories from expectedDirs
func filterEmptyDirs ( t * testing . T , items [ ] Item , expectedDirs [ ] string ) ( newExpectedDirs [ ] string ) {
2020-06-19 01:45:39 +08:00
dirs := map [ string ] struct { } { "" : { } }
2017-08-09 22:51:27 +08:00
for _ , item := range items {
base := item . Path
for {
base = path . Dir ( base )
if base == "." || base == "/" {
break
}
dirs [ base ] = struct { } { }
}
}
for _ , expectedDir := range expectedDirs {
if _ , found := dirs [ expectedDir ] ; found {
newExpectedDirs = append ( newExpectedDirs , expectedDir )
} else {
t . Logf ( "Filtering empty directory %q" , expectedDir )
}
}
return newExpectedDirs
}
2019-08-09 05:42:43 +08:00
// CheckListingWithRoot checks the fs to see if it has the
2015-09-23 01:47:16 +08:00
// expected contents with the given precision.
2016-11-27 19:49:31 +08:00
//
// If expectedDirs is non nil then we check those too. Note that no
// directories returned is also OK as some remotes don't return
// directories.
2019-08-09 05:42:43 +08:00
//
// dir is the directory used for the listing.
func CheckListingWithRoot ( t * testing . T , f fs . Fs , dir string , items [ ] Item , expectedDirs [ ] string , precision time . Duration ) {
2017-08-09 22:51:27 +08:00
if expectedDirs != nil && ! f . Features ( ) . CanHaveEmptyDirectories {
expectedDirs = filterEmptyDirs ( t , items , expectedDirs )
}
2014-07-25 05:50:11 +08:00
is := NewItems ( items )
2019-07-18 18:13:54 +08:00
ctx := context . Background ( )
oldErrors := accounting . Stats ( ctx ) . GetErrors ( )
2015-09-15 04:01:25 +08:00
var objs [ ] fs . Object
2017-06-30 20:37:29 +08:00
var dirs [ ] fs . Directory
2016-04-22 03:06:21 +08:00
var err error
2024-04-23 07:34:20 +08:00
retries := * ListRetries
2016-01-17 18:08:28 +08:00
sleep := time . Second / 2
2018-11-02 20:16:13 +08:00
wantListing := makeListingFromItems ( items )
2017-01-20 01:25:57 +08:00
gotListing := "<unset>"
2017-01-21 01:12:05 +08:00
listingOK := false
2015-11-14 20:57:17 +08:00
for i := 1 ; i <= retries ; i ++ {
2019-08-09 05:42:43 +08:00
objs , dirs , err = walk . GetAll ( ctx , f , dir , true , - 1 )
2016-04-22 03:06:21 +08:00
if err != nil && err != fs . ErrorDirNotFound {
t . Fatalf ( "Error listing: %v" , err )
2015-09-15 04:01:25 +08:00
}
2017-01-20 01:25:57 +08:00
gotListing = makeListingFromObjects ( objs )
2023-07-22 17:46:21 +08:00
2018-11-02 20:16:13 +08:00
listingOK = wantListing == gotListing
2017-08-09 22:51:27 +08:00
if listingOK && ( expectedDirs == nil || len ( dirs ) == len ( expectedDirs ) ) {
2016-01-17 18:08:28 +08:00
// Put an extra sleep in if we did any retries just to make sure it really
2024-01-03 19:49:46 +08:00
// is consistent
2016-01-17 18:08:28 +08:00
if i != 1 {
extraSleep := 5 * time . Second + sleep
t . Logf ( "Sleeping for %v just to make sure" , extraSleep )
time . Sleep ( extraSleep )
}
2015-09-15 04:01:25 +08:00
break
}
2016-01-17 18:08:28 +08:00
sleep *= 2
t . Logf ( "Sleeping for %v for list eventual consistency: %d/%d" , sleep , i , retries )
time . Sleep ( sleep )
2017-01-14 01:21:47 +08:00
if doDirCacheFlush := f . Features ( ) . DirCacheFlush ; doDirCacheFlush != nil {
2016-12-09 23:39:29 +08:00
t . Logf ( "Flushing the directory cache" )
2017-01-14 01:21:47 +08:00
doDirCacheFlush ( )
2016-12-09 23:39:29 +08:00
}
2015-09-15 04:01:25 +08:00
}
2018-11-02 20:16:13 +08:00
assert . True ( t , listingOK , fmt . Sprintf ( "listing wrong, want\n %s got\n %s" , wantListing , gotListing ) )
2015-09-15 04:01:25 +08:00
for _ , obj := range objs {
2016-06-30 00:59:31 +08:00
require . NotNil ( t , obj )
2014-08-02 00:58:39 +08:00
is . Find ( t , obj , precision )
2014-07-25 05:50:11 +08:00
}
2014-08-02 00:58:39 +08:00
is . Done ( t )
2015-08-25 04:42:23 +08:00
// Don't notice an error when listing an empty directory
2019-07-18 18:13:54 +08:00
if len ( items ) == 0 && oldErrors == 0 && accounting . Stats ( ctx ) . GetErrors ( ) == 1 {
accounting . Stats ( ctx ) . ResetErrors ( )
2015-08-25 04:42:23 +08:00
}
2017-08-09 22:51:27 +08:00
// Check the directories
if expectedDirs != nil {
2017-09-17 21:05:33 +08:00
expectedDirsCopy := make ( [ ] string , len ( expectedDirs ) )
for i , dir := range expectedDirs {
2019-08-27 03:22:38 +08:00
expectedDirsCopy [ i ] = Normalize ( dir )
2017-09-17 21:05:33 +08:00
}
2016-11-26 05:52:43 +08:00
actualDirs := [ ] string { }
for _ , dir := range dirs {
2019-08-27 03:22:38 +08:00
actualDirs = append ( actualDirs , Normalize ( dir . Remote ( ) ) )
2016-11-26 05:52:43 +08:00
}
sort . Strings ( actualDirs )
2017-09-17 21:05:33 +08:00
sort . Strings ( expectedDirsCopy )
assert . Equal ( t , expectedDirsCopy , actualDirs , "directories" )
2016-11-26 05:52:43 +08:00
}
2014-08-02 00:58:39 +08:00
}
2019-08-09 05:42:43 +08:00
// CheckListingWithPrecision checks the fs to see if it has the
// expected contents with the given precision.
//
// If expectedDirs is non nil then we check those too. Note that no
// directories returned is also OK as some remotes don't return
// directories.
func CheckListingWithPrecision ( t * testing . T , f fs . Fs , items [ ] Item , expectedDirs [ ] string , precision time . Duration ) {
CheckListingWithRoot ( t , f , "" , items , expectedDirs , precision )
}
2015-09-23 01:47:16 +08:00
// CheckListing checks the fs to see if it has the expected contents
2014-08-02 00:58:39 +08:00
func CheckListing ( t * testing . T , f fs . Fs , items [ ] Item ) {
precision := f . Precision ( )
2016-11-26 05:52:43 +08:00
CheckListingWithPrecision ( t , f , items , nil , precision )
2014-07-25 05:50:11 +08:00
}
2021-11-09 19:43:36 +08:00
// CheckItemsWithPrecision checks the fs with the specified precision
// to see if it has the expected items.
func CheckItemsWithPrecision ( t * testing . T , f fs . Fs , precision time . Duration , items ... Item ) {
CheckListingWithPrecision ( t , f , items , nil , precision )
}
2016-01-17 18:08:28 +08:00
// CheckItems checks the fs to see if it has only the items passed in
// using a precision of fs.Config.ModifyWindow
func CheckItems ( t * testing . T , f fs . Fs , items ... Item ) {
2020-11-06 00:27:01 +08:00
CheckListingWithPrecision ( t , f , items , nil , fs . GetModifyWindow ( context . TODO ( ) , f ) )
2016-01-17 18:08:28 +08:00
}
2019-08-17 17:28:44 +08:00
// CompareItems compares a set of DirEntries to a slice of items and a list of dirs
// The modtimes are compared with the precision supplied
func CompareItems ( t * testing . T , entries fs . DirEntries , items [ ] Item , expectedDirs [ ] string , precision time . Duration , what string ) {
2019-08-05 19:43:39 +08:00
is := NewItems ( items )
var objs [ ] fs . Object
var dirs [ ] fs . Directory
2018-11-02 20:16:13 +08:00
wantListing := makeListingFromItems ( items )
2019-08-05 19:43:39 +08:00
for _ , entry := range entries {
switch x := entry . ( type ) {
case fs . Directory :
dirs = append ( dirs , x )
case fs . Object :
objs = append ( objs , x )
// do nothing
default :
t . Fatalf ( "unknown object type %T" , entry )
}
}
gotListing := makeListingFromObjects ( objs )
2018-11-02 20:16:13 +08:00
listingOK := wantListing == gotListing
assert . True ( t , listingOK , fmt . Sprintf ( "%s not equal, want\n %s got\n %s" , what , wantListing , gotListing ) )
2019-08-05 19:43:39 +08:00
for _ , obj := range objs {
require . NotNil ( t , obj )
is . Find ( t , obj , precision )
}
is . Done ( t )
// Check the directories
if expectedDirs != nil {
expectedDirsCopy := make ( [ ] string , len ( expectedDirs ) )
for i , dir := range expectedDirs {
2019-08-27 03:22:38 +08:00
expectedDirsCopy [ i ] = Normalize ( dir )
2019-08-05 19:43:39 +08:00
}
actualDirs := [ ] string { }
for _ , dir := range dirs {
2019-08-27 03:22:38 +08:00
actualDirs = append ( actualDirs , Normalize ( dir . Remote ( ) ) )
2019-08-05 19:43:39 +08:00
}
sort . Strings ( actualDirs )
sort . Strings ( expectedDirsCopy )
assert . Equal ( t , expectedDirsCopy , actualDirs , "directories not equal" )
}
}
2015-09-23 01:47:16 +08:00
// Time parses a time string or logs a fatal error
2014-07-25 05:50:11 +08:00
func Time ( timeString string ) time . Time {
t , err := time . Parse ( time . RFC3339Nano , timeString )
if err != nil {
2024-08-18 22:58:35 +08:00
fs . Fatalf ( nil , "Failed to parse time %q: %v" , timeString , err )
2014-07-25 05:50:11 +08:00
}
return t
}
2015-09-23 01:47:16 +08:00
// LocalRemote creates a temporary directory name for local remotes
2014-08-01 04:24:52 +08:00
func LocalRemote ( ) ( path string , err error ) {
2022-08-20 22:38:02 +08:00
path , err = os . MkdirTemp ( "" , "rclone" )
2014-08-01 04:24:52 +08:00
if err == nil {
// Now remove the directory
err = os . Remove ( path )
}
2015-02-07 23:52:06 +08:00
path = filepath . ToSlash ( path )
2014-08-01 04:24:52 +08:00
return
}
2015-09-23 01:47:16 +08:00
// RandomRemoteName makes a random bucket or subdirectory name
2014-07-25 05:50:11 +08:00
//
2014-08-01 04:24:52 +08:00
// Returns a random remote name plus the leaf name
func RandomRemoteName ( remoteName string ) ( string , string , error ) {
2014-07-25 05:50:11 +08:00
var err error
2014-08-01 04:24:52 +08:00
var leafName string
// Make a directory if remote name is null
2014-07-25 05:50:11 +08:00
if remoteName == "" {
2014-08-01 04:24:52 +08:00
remoteName , err = LocalRemote ( )
2014-07-25 05:50:11 +08:00
if err != nil {
2014-08-01 04:24:52 +08:00
return "" , "" , err
}
} else {
if ! strings . HasSuffix ( remoteName , ":" ) {
remoteName += "/"
2014-07-25 05:50:11 +08:00
}
2024-04-23 07:34:20 +08:00
leafName = "rclone-test-" + random . String ( 12 )
2016-01-24 20:37:46 +08:00
if ! MatchTestRemote . MatchString ( leafName ) {
2024-08-18 22:58:35 +08:00
fs . Fatalf ( nil , "%q didn't match the test remote name regexp" , leafName )
2016-01-24 20:37:46 +08:00
}
2014-08-01 04:24:52 +08:00
remoteName += leafName
2014-07-25 05:50:11 +08:00
}
2014-08-01 04:24:52 +08:00
return remoteName , leafName , nil
}
2014-07-25 05:50:11 +08:00
2015-09-23 01:47:16 +08:00
// RandomRemote makes a random bucket or subdirectory on the remote
2019-08-09 02:58:02 +08:00
// from the -remote parameter
2014-08-01 04:24:52 +08:00
//
// Call the finalise function returned to Purge the fs at the end (and
// the parent if necessary)
2016-07-11 18:36:46 +08:00
//
// Returns the remote, its url, a finaliser and an error
2019-08-09 02:58:02 +08:00
func RandomRemote ( ) ( fs . Fs , string , func ( ) , error ) {
2014-08-01 04:24:52 +08:00
var err error
2014-07-25 05:50:11 +08:00
var parentRemote fs . Fs
2019-08-09 02:58:02 +08:00
remoteName := * RemoteName
2014-08-01 04:24:52 +08:00
remoteName , _ , err = RandomRemoteName ( remoteName )
if err != nil {
2016-07-11 18:36:46 +08:00
return nil , "" , nil , err
2014-08-01 04:24:52 +08:00
}
2020-11-05 23:18:51 +08:00
remote , err := fs . NewFs ( context . Background ( ) , remoteName )
2014-07-25 05:50:11 +08:00
if err != nil {
2016-07-11 18:36:46 +08:00
return nil , "" , nil , err
2014-07-25 05:50:11 +08:00
}
finalise := func ( ) {
2018-01-13 00:30:54 +08:00
Purge ( remote )
2014-07-25 05:50:11 +08:00
if parentRemote != nil {
2018-01-13 00:30:54 +08:00
Purge ( parentRemote )
2014-07-29 04:02:00 +08:00
if err != nil {
2024-08-18 22:58:35 +08:00
fs . Logf ( nil , "Failed to purge %v: %v" , parentRemote , err )
2014-07-29 04:02:00 +08:00
}
2014-07-25 05:50:11 +08:00
}
}
2016-07-11 18:36:46 +08:00
return remote , remoteName , finalise , nil
2014-07-25 05:50:11 +08:00
}
2018-01-13 00:30:54 +08:00
// Purge is a simplified re-implementation of operations.Purge for the
// test routine cleanup to avoid circular dependencies.
//
// It logs errors rather than returning them
func Purge ( f fs . Fs ) {
2019-06-17 16:34:30 +08:00
ctx := context . Background ( )
2018-01-13 00:30:54 +08:00
var err error
doFallbackPurge := true
if doPurge := f . Features ( ) . Purge ; doPurge != nil {
doFallbackPurge = false
2018-01-19 18:20:06 +08:00
fs . Debugf ( f , "Purge remote" )
2020-06-05 05:25:14 +08:00
err = doPurge ( ctx , "" )
2018-01-13 00:30:54 +08:00
if err == fs . ErrorCantPurge {
doFallbackPurge = true
}
}
if doFallbackPurge {
2018-01-19 18:20:06 +08:00
dirs := [ ] string { "" }
2019-06-17 16:34:30 +08:00
err = walk . ListR ( ctx , f , "" , true , - 1 , walk . ListAll , func ( entries fs . DirEntries ) error {
2019-01-22 01:02:48 +08:00
var err error
2018-01-13 00:30:54 +08:00
entries . ForObject ( func ( obj fs . Object ) {
2018-01-19 18:20:06 +08:00
fs . Debugf ( f , "Purge object %q" , obj . Remote ( ) )
2019-06-17 16:34:30 +08:00
err = obj . Remove ( ctx )
2018-01-13 00:30:54 +08:00
if err != nil {
2024-08-18 22:58:35 +08:00
fs . Logf ( nil , "purge failed to remove %q: %v" , obj . Remote ( ) , err )
2018-01-13 00:30:54 +08:00
}
} )
entries . ForDir ( func ( dir fs . Directory ) {
dirs = append ( dirs , dir . Remote ( ) )
} )
return nil
} )
sort . Strings ( dirs )
for i := len ( dirs ) - 1 ; i >= 0 ; i -- {
dir := dirs [ i ]
2018-01-19 18:20:06 +08:00
fs . Debugf ( f , "Purge dir %q" , dir )
2019-06-17 16:34:30 +08:00
err := f . Rmdir ( ctx , dir )
2018-01-13 00:30:54 +08:00
if err != nil {
2024-08-18 22:58:35 +08:00
fs . Logf ( nil , "purge failed to rmdir %q: %v" , dir , err )
2018-01-13 00:30:54 +08:00
}
}
}
if err != nil {
2024-08-18 22:58:35 +08:00
fs . Logf ( nil , "purge failed: %v" , err )
2018-01-13 00:30:54 +08:00
}
2014-07-25 05:50:11 +08:00
}
2024-02-22 19:13:32 +08:00
2024-02-26 21:44:55 +08:00
// NewObject finds the object on the remote
func NewObject ( ctx context . Context , t * testing . T , f fs . Fs , remote string ) fs . Object {
var obj fs . Object
var err error
sleepTime := 1 * time . Second
for i := 1 ; i <= * ListRetries ; i ++ {
obj , err = f . NewObject ( ctx , remote )
if err == nil {
break
}
t . Logf ( "Sleeping for %v for findObject eventual consistency: %d/%d (%v)" , sleepTime , i , * ListRetries , err )
time . Sleep ( sleepTime )
sleepTime = ( sleepTime * 3 ) / 2
}
require . NoError ( t , err )
return obj
}
2024-04-05 01:03:20 +08:00
// NewDirectoryRetries finds the directory with remote in f
//
// If directory can't be found it returns an error wrapping fs.ErrorDirNotFound
2024-02-22 19:13:32 +08:00
//
// One day this will be an rclone primitive
2024-04-05 01:03:20 +08:00
func NewDirectoryRetries ( ctx context . Context , t * testing . T , f fs . Fs , remote string , retries int ) ( fs . Directory , error ) {
2024-02-22 19:13:32 +08:00
var err error
var dir fs . Directory
sleepTime := 1 * time . Second
root := path . Dir ( remote )
if root == "." {
root = ""
}
2024-04-05 01:03:20 +08:00
for i := 1 ; i <= retries ; i ++ {
2024-02-22 19:13:32 +08:00
var entries fs . DirEntries
entries , err = f . List ( ctx , root )
if err != nil {
continue
}
for _ , entry := range entries {
var ok bool
dir , ok = entry . ( fs . Directory )
if ok && dir . Remote ( ) == remote {
2024-04-05 01:03:20 +08:00
return dir , nil
2024-02-22 19:13:32 +08:00
}
}
2024-04-05 01:03:20 +08:00
err = fmt . Errorf ( "directory %q not found in %q: %w" , remote , root , fs . ErrorDirNotFound )
if i < retries {
t . Logf ( "Sleeping for %v for NewDirectoryRetries eventual consistency: %d/%d (%v)" , sleepTime , i , retries , err )
time . Sleep ( sleepTime )
sleepTime = ( sleepTime * 3 ) / 2
}
2024-02-22 19:13:32 +08:00
}
2024-04-05 01:03:20 +08:00
return dir , err
}
// NewDirectory finds the directory with remote in f
//
// One day this will be an rclone primitive
func NewDirectory ( ctx context . Context , t * testing . T , f fs . Fs , remote string ) fs . Directory {
dir , err := NewDirectoryRetries ( ctx , t , f , remote , * ListRetries )
2024-02-22 19:13:32 +08:00
require . NoError ( t , err )
return dir
}
// CheckEntryMetadata checks the metadata on the directory
//
// This checks a limited set of metadata on the directory
func CheckEntryMetadata ( ctx context . Context , t * testing . T , f fs . Fs , entry fs . DirEntry , wantMeta fs . Metadata ) {
features := f . Features ( )
do , ok := entry . ( fs . Metadataer )
require . True ( t , ok , "Didn't find expected Metadata() method on %T" , entry )
gotMeta , err := do . Metadata ( ctx )
require . NoError ( t , err )
for k , v := range wantMeta {
switch k {
case "mtime" , "atime" , "btime" , "ctime" :
// Check the system time Metadata
wantT , err := time . Parse ( time . RFC3339 , v )
require . NoError ( t , err )
gotT , err := time . Parse ( time . RFC3339 , gotMeta [ k ] )
require . NoError ( t , err )
AssertTimeEqualWithPrecision ( t , entry . Remote ( ) , wantT , gotT , f . Precision ( ) )
default :
// Check the User metadata if we can
_ , isDir := entry . ( fs . Directory )
if ( isDir && features . UserDirMetadata ) || ( ! isDir && features . UserMetadata ) {
assert . Equal ( t , v , gotMeta [ k ] )
}
}
}
}
// CheckDirModTime checks the modtime on the directory
func CheckDirModTime ( ctx context . Context , t * testing . T , f fs . Fs , dir fs . Directory , wantT time . Time ) {
2024-03-01 18:56:48 +08:00
if f . Features ( ) . DirSetModTime == nil && f . Features ( ) . MkdirMetadata == nil {
fs . Debugf ( f , "Skipping modtime test as remote does not support DirSetModTime or MkdirMetadata" )
return
}
2024-02-22 19:13:32 +08:00
gotT := dir . ModTime ( ctx )
2024-04-24 18:38:22 +08:00
precision := f . Precision ( )
// For unknown reasons the precision of modification times of
// directories on the CI is about >15mS. The tests work fine
// when run in Virtualbox though so I conjecture this is
// something to do with the file system used there.
if runtime . GOOS == "windows" && testy . CI ( ) {
precision = 100 * time . Millisecond
}
AssertTimeEqualWithPrecision ( t , dir . Remote ( ) , wantT , gotT , precision )
2024-02-22 19:13:32 +08:00
}
2024-03-01 08:49:34 +08:00
// Gz returns a compressed version of its input string
func Gz ( t * testing . T , s string ) string {
var buf bytes . Buffer
zw := gzip . NewWriter ( & buf )
_ , err := zw . Write ( [ ] byte ( s ) )
require . NoError ( t , err )
err = zw . Close ( )
require . NoError ( t , err )
return buf . String ( )
}