vfs: make objects of unknown size readable through the VFS

These objects (eg Google Docs) appear with 0 length in the VFS.

Before this change, these only read 0 bytes.

After this change, even though the size appears to be 0, the objects
can be read to the end.  If the objects are read to the end then the
size on the handle will be updated.

vfs/file_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/mockfs"
+	"github.com/rclone/rclone/fstest/mockobject"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -110,6 +112,51 @@ func TestFileOpenRead(t *testing.T) {
 	require.NoError(t, fd.Close())
 }
 
+func TestFileOpenReadUnknownSize(t *testing.T) {
+	var (
+		contents = []byte("file contents")
+		remote   = "file.txt"
+		ctx      = context.Background()
+	)
+
+	// create a mock object which returns size -1
+	o := mockobject.New(remote).WithContent(contents, mockobject.SeekModeNone)
+	o.SetUnknownSize(true)
+	assert.Equal(t, int64(-1), o.Size())
+
+	// add it to a mock fs
+	f := mockfs.NewFs("test", "root")
+	f.AddObject(o)
+	testObj, err := f.NewObject(ctx, remote)
+	require.NoError(t, err)
+	assert.Equal(t, int64(-1), testObj.Size())
+
+	// create a VFS from that mockfs
+	vfs := New(f, nil)
+
+	// find the file
+	node, err := vfs.Stat(remote)
+	require.NoError(t, err)
+	require.True(t, node.IsFile())
+	file := node.(*File)
+
+	// open it
+	fd, err := file.openRead()
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), fd.Size())
+
+	// check the contents are not empty even though size is empty
+	gotContents, err := ioutil.ReadAll(fd)
+	require.NoError(t, err)
+	assert.Equal(t, contents, gotContents)
+	t.Logf("gotContents = %q", gotContents)
+
+	// check that file size has been updated
+	assert.Equal(t, int64(len(contents)), fd.Size())
+
+	require.NoError(t, fd.Close())
+}
+
 func TestFileOpenWrite(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()

vfs/read.go

@@ -16,19 +16,20 @@ import (
 // ReadFileHandle is an open for read file handle on a File
 type ReadFileHandle struct {
 	baseHandle
-	done       func(err error)
+	done        func(err error)
-	mu         sync.Mutex
+	mu          sync.Mutex
-	closed     bool // set if handle has been closed
+	closed      bool // set if handle has been closed
-	r          *accounting.Account
+	r           *accounting.Account
-	readCalled bool  // set if read has been called
+	readCalled  bool  // set if read has been called
-	size       int64 // size of the object
+	size        int64 // size of the object (0 for unknown length)
-	offset     int64 // offset of read of o
+	offset      int64 // offset of read of o
-	roffset    int64 // offset of Read() calls
+	roffset     int64 // offset of Read() calls
-	noSeek     bool
+	noSeek      bool
-	file       *File
+	sizeUnknown bool // set if size of source is not known
-	hash       *hash.MultiHasher
+	file        *File
-	opened     bool
+	hash        *hash.MultiHasher
-	remote     string
+	opened      bool
+	remote      string
 }
 
 // Check interfaces
@@ -51,11 +52,12 @@ func newReadFileHandle(f *File) (*ReadFileHandle, error) {
 	}
 
 	fh := &ReadFileHandle{
-		remote: o.Remote(),
+		remote:      o.Remote(),
-		noSeek: f.d.vfs.Opt.NoSeek,
+		noSeek:      f.d.vfs.Opt.NoSeek,
-		file:   f,
+		file:        f,
-		hash:   mhash,
+		hash:        mhash,
-		size:   nonNegative(o.Size()),
+		size:        nonNegative(o.Size()),
+		sizeUnknown: o.Size() < 0,
 	}
 	return fh, nil
 }
@@ -208,6 +210,7 @@ func (fh *ReadFileHandle) ReadAt(p []byte, off int64) (n int, err error) {
 
 // Implementation of ReadAt - call with lock held
 func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
+	// defer log.Trace(fh.remote, "p[%d], off=%d", len(p), off)("n=%d, err=%v", &n, &err)
 	err = fh.openPending() // FIXME pending open could be more efficient in the presense of seek (and retries)
 	if err != nil {
 		return 0, err
@@ -250,7 +253,12 @@ func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
 			// }
 			if err == nil {
 				break
-			} else if (err == io.ErrUnexpectedEOF || err == io.EOF) && newOffset == fh.size {
+			} else if (err == io.ErrUnexpectedEOF || err == io.EOF) && (newOffset == fh.size || fh.sizeUnknown) {
+				if fh.sizeUnknown {
+					// size is now known since we have read to the end
+					fh.sizeUnknown = false
+					fh.size = newOffset
+				}
 				// Have read to end of file - reset error
 				err = nil
 				break
@@ -331,7 +339,7 @@ func (fh *ReadFileHandle) checkHash() error {
 func (fh *ReadFileHandle) Read(p []byte) (n int, err error) {
 	fh.mu.Lock()
 	defer fh.mu.Unlock()
-	if fh.roffset >= fh.size {
+	if fh.roffset >= fh.size && !fh.sizeUnknown {
 		return 0, io.EOF
 	}
 	n, err = fh.readAt(p, fh.roffset)