From 42d997f639e1c4087c68d1e75cf76aa847d6ac5f Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 7 Jan 2019 14:47:19 +0000 Subject: [PATCH] lib/file: reimplement os.OpenFile allowing rename/delete open files under Windows Normally os.OpenFile under Windows does not allow renaming or deleting open file handles. This package provides equivelents for os.OpenFile, os.Open and os.Create which do allow that. --- lib/file/file.go | 22 ++++++ lib/file/file_other.go | 15 ++++ lib/file/file_test.go | 154 +++++++++++++++++++++++++++++++++++++++ lib/file/file_windows.go | 66 +++++++++++++++++ 4 files changed, 257 insertions(+) create mode 100644 lib/file/file.go create mode 100644 lib/file/file_other.go create mode 100644 lib/file/file_test.go create mode 100644 lib/file/file_windows.go diff --git a/lib/file/file.go b/lib/file/file.go new file mode 100644 index 000000000..a82c144e8 --- /dev/null +++ b/lib/file/file.go @@ -0,0 +1,22 @@ +// Package file provides a version of os.OpenFile, the handles of +// which can be renamed and deleted under Windows. +package file + +import "os" + +// Open opens the named file for reading. If successful, methods on +// the returned file can be used for reading; the associated file +// descriptor has mode O_RDONLY. +// If there is an error, it will be of type *PathError. +func Open(name string) (*os.File, error) { + return OpenFile(name, os.O_RDONLY, 0) +} + +// Create creates the named file with mode 0666 (before umask), truncating +// it if it already exists. If successful, methods on the returned +// File can be used for I/O; the associated file descriptor has mode +// O_RDWR. +// If there is an error, it will be of type *PathError. +func Create(name string) (*os.File, error) { + return OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) +} diff --git a/lib/file/file_other.go b/lib/file/file_other.go new file mode 100644 index 000000000..16de15847 --- /dev/null +++ b/lib/file/file_other.go @@ -0,0 +1,15 @@ +//+build !windows + +package file + +import "os" + +// OpenFile is the generalized open call; most users will use Open or Create +// instead. It opens the named file with specified flag (O_RDONLY etc.) and +// perm (before umask), if applicable. If successful, methods on the returned +// File can be used for I/O. If there is an error, it will be of type +// *PathError. +// +// Under both Unix and Windows this will allow open files to be +// renamed and or deleted. +var OpenFile = os.OpenFile diff --git a/lib/file/file_test.go b/lib/file/file_test.go new file mode 100644 index 000000000..0f4e05121 --- /dev/null +++ b/lib/file/file_test.go @@ -0,0 +1,154 @@ +package file + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Create a test directory then tidy up +func testDir(t *testing.T) (string, func()) { + dir, err := ioutil.TempDir("", "rclone-test") + require.NoError(t, err) + return dir, func() { + assert.NoError(t, os.RemoveAll(dir)) + } +} + +// This lists dir and checks the listing is as expected without checking the size +func checkListingNoSize(t *testing.T, dir string, want []string) { + var got []string + nodes, err := ioutil.ReadDir(dir) + require.NoError(t, err) + for _, node := range nodes { + got = append(got, fmt.Sprintf("%s,%v", node.Name(), node.IsDir())) + } + assert.Equal(t, want, got) +} + +// This lists dir and checks the listing is as expected +func checkListing(t *testing.T, dir string, want []string) { + var got []string + nodes, err := ioutil.ReadDir(dir) + require.NoError(t, err) + for _, node := range nodes { + got = append(got, fmt.Sprintf("%s,%d,%v", node.Name(), node.Size(), node.IsDir())) + } + assert.Equal(t, want, got) +} + +// Test we can rename an open file +func TestOpenFileRename(t *testing.T) { + dir, tidy := testDir(t) + defer tidy() + + filepath := path.Join(dir, "file1") + f, err := Create(filepath) + require.NoError(t, err) + + _, err = f.Write([]byte("hello")) + assert.NoError(t, err) + + checkListingNoSize(t, dir, []string{ + "file1,false", + }) + + // Delete the file first + assert.NoError(t, os.Remove(filepath)) + + // .. then close it + assert.NoError(t, f.Close()) + + checkListing(t, dir, nil) +} + +// Test we can delete an open file +func TestOpenFileDelete(t *testing.T) { + dir, tidy := testDir(t) + defer tidy() + + filepath := path.Join(dir, "file1") + f, err := Create(filepath) + require.NoError(t, err) + + _, err = f.Write([]byte("hello")) + assert.NoError(t, err) + + checkListingNoSize(t, dir, []string{ + "file1,false", + }) + + // Rename the file while open + filepath2 := path.Join(dir, "file2") + assert.NoError(t, os.Rename(filepath, filepath2)) + + checkListingNoSize(t, dir, []string{ + "file2,false", + }) + + // .. then close it + assert.NoError(t, f.Close()) + + checkListing(t, dir, []string{ + "file2,5,false", + }) +} + +// Smoke test the Open, OpenFile and Create functions +func TestOpenFileOperations(t *testing.T) { + dir, tidy := testDir(t) + defer tidy() + + filepath := path.Join(dir, "file1") + + // Create the file + + f, err := Create(filepath) + require.NoError(t, err) + + _, err = f.Write([]byte("hello")) + assert.NoError(t, err) + + assert.NoError(t, f.Close()) + + checkListing(t, dir, []string{ + "file1,5,false", + }) + + // Append onto the file + + f, err = OpenFile(filepath, os.O_RDWR|os.O_APPEND, 0666) + require.NoError(t, err) + + _, err = f.Write([]byte("HI")) + assert.NoError(t, err) + + assert.NoError(t, f.Close()) + + checkListing(t, dir, []string{ + "file1,7,false", + }) + + // Read it back in + + f, err = Open(filepath) + require.NoError(t, err) + var b = make([]byte, 10) + n, err := f.Read(b) + assert.True(t, err == io.EOF || err == nil) + assert.Equal(t, 7, n) + assert.Equal(t, "helloHI", string(b[:n])) + + assert.NoError(t, f.Close()) + + checkListing(t, dir, []string{ + "file1,7,false", + }) + +} diff --git a/lib/file/file_windows.go b/lib/file/file_windows.go new file mode 100644 index 000000000..29a6fad0f --- /dev/null +++ b/lib/file/file_windows.go @@ -0,0 +1,66 @@ +//+build windows + +package file + +import ( + "os" + "syscall" +) + +// OpenFile is the generalized open call; most users will use Open or Create +// instead. It opens the named file with specified flag (O_RDONLY etc.) and +// perm (before umask), if applicable. If successful, methods on the returned +// File can be used for I/O. If there is an error, it will be of type +// *PathError. +// +// Under both Unix and Windows this will allow open files to be +// renamed and or deleted. +func OpenFile(path string, mode int, perm os.FileMode) (*os.File, error) { + // This code copied from syscall_windows.go in the go source and then + // modified to support renaming and deleting open files by adding + // FILE_SHARE_DELETE. + // + // https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-createfilea#file_share_delete + if len(path) == 0 { + return nil, syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + var access uint32 + switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { + case syscall.O_RDONLY: + access = syscall.GENERIC_READ + case syscall.O_WRONLY: + access = syscall.GENERIC_WRITE + case syscall.O_RDWR: + access = syscall.GENERIC_READ | syscall.GENERIC_WRITE + } + if mode&syscall.O_CREAT != 0 { + access |= syscall.GENERIC_WRITE + } + if mode&syscall.O_APPEND != 0 { + access &^= syscall.GENERIC_WRITE + access |= syscall.FILE_APPEND_DATA + } + sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE) + var createmode uint32 + switch { + case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): + createmode = syscall.CREATE_NEW + case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC): + createmode = syscall.CREATE_ALWAYS + case mode&syscall.O_CREAT == syscall.O_CREAT: + createmode = syscall.OPEN_ALWAYS + case mode&syscall.O_TRUNC == syscall.O_TRUNC: + createmode = syscall.TRUNCATE_EXISTING + default: + createmode = syscall.OPEN_EXISTING + } + h, e := syscall.CreateFile(pathp, access, sharemode, nil, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0) + if e != nil { + return nil, e + } + return os.NewFile(uintptr(h), path), nil +}