vfs: make objects of unknown size readable through the VFS

These objects (eg Google Docs) appear with 0 length in the VFS.

Before this change, these only read 0 bytes.

After this change, even though the size appears to be 0, the objects
can be read to the end.  If the objects are read to the end then the
size on the handle will be updated.

vfs/file_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/mockfs"
+	"github.com/rclone/rclone/fstest/mockobject"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -110,6 +112,51 @@ func TestFileOpenRead(t *testing.T) {
 	require.NoError(t, fd.Close())
 }
 
+func TestFileOpenReadUnknownSize(t *testing.T) {
+	var (
+		contents = []byte("file contents")
+		remote   = "file.txt"
+		ctx      = context.Background()
+	)
+
+	// create a mock object which returns size -1
+	o := mockobject.New(remote).WithContent(contents, mockobject.SeekModeNone)
+	o.SetUnknownSize(true)
+	assert.Equal(t, int64(-1), o.Size())
+
+	// add it to a mock fs
+	f := mockfs.NewFs("test", "root")
+	f.AddObject(o)
+	testObj, err := f.NewObject(ctx, remote)
+	require.NoError(t, err)
+	assert.Equal(t, int64(-1), testObj.Size())
+
+	// create a VFS from that mockfs
+	vfs := New(f, nil)
+
+	// find the file
+	node, err := vfs.Stat(remote)
+	require.NoError(t, err)
+	require.True(t, node.IsFile())
+	file := node.(*File)
+
+	// open it
+	fd, err := file.openRead()
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), fd.Size())
+
+	// check the contents are not empty even though size is empty
+	gotContents, err := ioutil.ReadAll(fd)
+	require.NoError(t, err)
+	assert.Equal(t, contents, gotContents)
+	t.Logf("gotContents = %q", gotContents)
+
+	// check that file size has been updated
+	assert.Equal(t, int64(len(contents)), fd.Size())
+
+	require.NoError(t, fd.Close())
+}
+
 func TestFileOpenWrite(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()

vfs/read.go

@@ -21,10 +21,11 @@ type ReadFileHandle struct {
 	closed      bool // set if handle has been closed
 	r           *accounting.Account
 	readCalled  bool  // set if read has been called
-	size       int64 // size of the object
+	size        int64 // size of the object (0 for unknown length)
 	offset      int64 // offset of read of o
 	roffset     int64 // offset of Read() calls
 	noSeek      bool
+	sizeUnknown bool // set if size of source is not known
 	file        *File
 	hash        *hash.MultiHasher
 	opened      bool
@@ -56,6 +57,7 @@ func newReadFileHandle(f *File) (*ReadFileHandle, error) {
 		file:        f,
 		hash:        mhash,
 		size:        nonNegative(o.Size()),
+		sizeUnknown: o.Size() < 0,
 	}
 	return fh, nil
 }
@@ -208,6 +210,7 @@ func (fh *ReadFileHandle) ReadAt(p []byte, off int64) (n int, err error) {
 
 // Implementation of ReadAt - call with lock held
 func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
+	// defer log.Trace(fh.remote, "p[%d], off=%d", len(p), off)("n=%d, err=%v", &n, &err)
 	err = fh.openPending() // FIXME pending open could be more efficient in the presense of seek (and retries)
 	if err != nil {
 		return 0, err
@@ -250,7 +253,12 @@ func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
 			// }
 			if err == nil {
 				break
-			} else if (err == io.ErrUnexpectedEOF || err == io.EOF) && newOffset == fh.size {
+			} else if (err == io.ErrUnexpectedEOF || err == io.EOF) && (newOffset == fh.size || fh.sizeUnknown) {
+				if fh.sizeUnknown {
+					// size is now known since we have read to the end
+					fh.sizeUnknown = false
+					fh.size = newOffset
+				}
 				// Have read to end of file - reset error
 				err = nil
 				break
@@ -331,7 +339,7 @@ func (fh *ReadFileHandle) checkHash() error {
 func (fh *ReadFileHandle) Read(p []byte) (n int, err error) {
 	fh.mu.Lock()
 	defer fh.mu.Unlock()
-	if fh.roffset >= fh.size {
+	if fh.roffset >= fh.size && !fh.sizeUnknown {
 		return 0, io.EOF
 	}
 	n, err = fh.readAt(p, fh.roffset)