lib/file: reimplement os.OpenFile allowing rename/delete open files under Windows

Normally os.OpenFile under Windows does not allow renaming or deleting
open file handles.  This package provides equivelents for os.OpenFile,
os.Open and os.Create which do allow that.

lib/file/file.go

@@ -0,0 +1,22 @@
+// Package file provides a version of os.OpenFile, the handles of
+// which can be renamed and deleted under Windows.
+package file
+
+import "os"
+
+// Open opens the named file for reading. If successful, methods on
+// the returned file can be used for reading; the associated file
+// descriptor has mode O_RDONLY.
+// If there is an error, it will be of type *PathError.
+func Open(name string) (*os.File, error) {
+	return OpenFile(name, os.O_RDONLY, 0)
+}
+
+// Create creates the named file with mode 0666 (before umask), truncating
+// it if it already exists. If successful, methods on the returned
+// File can be used for I/O; the associated file descriptor has mode
+// O_RDWR.
+// If there is an error, it will be of type *PathError.
+func Create(name string) (*os.File, error) {
+	return OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+}

lib/file/file_other.go

@@ -0,0 +1,15 @@
+//+build !windows
+
+package file
+
+import "os"
+
+// OpenFile is the generalized open call; most users will use Open or Create
+// instead. It opens the named file with specified flag (O_RDONLY etc.) and
+// perm (before umask), if applicable. If successful, methods on the returned
+// File can be used for I/O. If there is an error, it will be of type
+// *PathError.
+//
+// Under both Unix and Windows this will allow open files to be
+// renamed and or deleted.
+var OpenFile = os.OpenFile

lib/file/file_test.go

@@ -0,0 +1,154 @@
+package file
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Create a test directory then tidy up
+func testDir(t *testing.T) (string, func()) {
+	dir, err := ioutil.TempDir("", "rclone-test")
+	require.NoError(t, err)
+	return dir, func() {
+		assert.NoError(t, os.RemoveAll(dir))
+	}
+}
+
+// This lists dir and checks the listing is as expected without checking the size
+func checkListingNoSize(t *testing.T, dir string, want []string) {
+	var got []string
+	nodes, err := ioutil.ReadDir(dir)
+	require.NoError(t, err)
+	for _, node := range nodes {
+		got = append(got, fmt.Sprintf("%s,%v", node.Name(), node.IsDir()))
+	}
+	assert.Equal(t, want, got)
+}
+
+// This lists dir and checks the listing is as expected
+func checkListing(t *testing.T, dir string, want []string) {
+	var got []string
+	nodes, err := ioutil.ReadDir(dir)
+	require.NoError(t, err)
+	for _, node := range nodes {
+		got = append(got, fmt.Sprintf("%s,%d,%v", node.Name(), node.Size(), node.IsDir()))
+	}
+	assert.Equal(t, want, got)
+}
+
+// Test we can rename an open file
+func TestOpenFileRename(t *testing.T) {
+	dir, tidy := testDir(t)
+	defer tidy()
+
+	filepath := path.Join(dir, "file1")
+	f, err := Create(filepath)
+	require.NoError(t, err)
+
+	_, err = f.Write([]byte("hello"))
+	assert.NoError(t, err)
+
+	checkListingNoSize(t, dir, []string{
+		"file1,false",
+	})
+
+	// Delete the file first
+	assert.NoError(t, os.Remove(filepath))
+
+	// .. then close it
+	assert.NoError(t, f.Close())
+
+	checkListing(t, dir, nil)
+}
+
+// Test we can delete an open file
+func TestOpenFileDelete(t *testing.T) {
+	dir, tidy := testDir(t)
+	defer tidy()
+
+	filepath := path.Join(dir, "file1")
+	f, err := Create(filepath)
+	require.NoError(t, err)
+
+	_, err = f.Write([]byte("hello"))
+	assert.NoError(t, err)
+
+	checkListingNoSize(t, dir, []string{
+		"file1,false",
+	})
+
+	// Rename the file while open
+	filepath2 := path.Join(dir, "file2")
+	assert.NoError(t, os.Rename(filepath, filepath2))
+
+	checkListingNoSize(t, dir, []string{
+		"file2,false",
+	})
+
+	// .. then close it
+	assert.NoError(t, f.Close())
+
+	checkListing(t, dir, []string{
+		"file2,5,false",
+	})
+}
+
+// Smoke test the Open, OpenFile and Create functions
+func TestOpenFileOperations(t *testing.T) {
+	dir, tidy := testDir(t)
+	defer tidy()
+
+	filepath := path.Join(dir, "file1")
+
+	// Create the file
+
+	f, err := Create(filepath)
+	require.NoError(t, err)
+
+	_, err = f.Write([]byte("hello"))
+	assert.NoError(t, err)
+
+	assert.NoError(t, f.Close())
+
+	checkListing(t, dir, []string{
+		"file1,5,false",
+	})
+
+	// Append onto the file
+
+	f, err = OpenFile(filepath, os.O_RDWR|os.O_APPEND, 0666)
+	require.NoError(t, err)
+
+	_, err = f.Write([]byte("HI"))
+	assert.NoError(t, err)
+
+	assert.NoError(t, f.Close())
+
+	checkListing(t, dir, []string{
+		"file1,7,false",
+	})
+
+	// Read it back in
+
+	f, err = Open(filepath)
+	require.NoError(t, err)
+	var b = make([]byte, 10)
+	n, err := f.Read(b)
+	assert.True(t, err == io.EOF || err == nil)
+	assert.Equal(t, 7, n)
+	assert.Equal(t, "helloHI", string(b[:n]))
+
+	assert.NoError(t, f.Close())
+
+	checkListing(t, dir, []string{
+		"file1,7,false",
+	})
+
+}

lib/file/file_windows.go

@@ -0,0 +1,66 @@
+//+build windows
+
+package file
+
+import (
+	"os"
+	"syscall"
+)
+
+// OpenFile is the generalized open call; most users will use Open or Create
+// instead. It opens the named file with specified flag (O_RDONLY etc.) and
+// perm (before umask), if applicable. If successful, methods on the returned
+// File can be used for I/O. If there is an error, it will be of type
+// *PathError.
+//
+// Under both Unix and Windows this will allow open files to be
+// renamed and or deleted.
+func OpenFile(path string, mode int, perm os.FileMode) (*os.File, error) {
+	// This code copied from syscall_windows.go in the go source and then
+	// modified to support renaming and deleting open files by adding
+	// FILE_SHARE_DELETE.
+	//
+	// https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-createfilea#file_share_delete
+	if len(path) == 0 {
+		return nil, syscall.ERROR_FILE_NOT_FOUND
+	}
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return nil, err
+	}
+	var access uint32
+	switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) {
+	case syscall.O_RDONLY:
+		access = syscall.GENERIC_READ
+	case syscall.O_WRONLY:
+		access = syscall.GENERIC_WRITE
+	case syscall.O_RDWR:
+		access = syscall.GENERIC_READ | syscall.GENERIC_WRITE
+	}
+	if mode&syscall.O_CREAT != 0 {
+		access |= syscall.GENERIC_WRITE
+	}
+	if mode&syscall.O_APPEND != 0 {
+		access &^= syscall.GENERIC_WRITE
+		access |= syscall.FILE_APPEND_DATA
+	}
+	sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE)
+	var createmode uint32
+	switch {
+	case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL):
+		createmode = syscall.CREATE_NEW
+	case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC):
+		createmode = syscall.CREATE_ALWAYS
+	case mode&syscall.O_CREAT == syscall.O_CREAT:
+		createmode = syscall.OPEN_ALWAYS
+	case mode&syscall.O_TRUNC == syscall.O_TRUNC:
+		createmode = syscall.TRUNCATE_EXISTING
+	default:
+		createmode = syscall.OPEN_EXISTING
+	}
+	h, e := syscall.CreateFile(pathp, access, sharemode, nil, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0)
+	if e != nil {
+		return nil, e
+	}
+	return os.NewFile(uintptr(h), path), nil
+}