cmd/gitannex: Add the gitannex subcommand

This commit adds a new subcommand named "gitannex", aka
"git-annex-remote-rclone-builtin" when invoked via a symlink.

This accomplishes milestone 1 from issue #7625: "minimal support for the
external special remote protocol".

Issue #7625

cmd/all/all.go

@@ -25,6 +25,7 @@ import (
 	_ "github.com/rclone/rclone/cmd/deletefile"
 	_ "github.com/rclone/rclone/cmd/genautocomplete"
 	_ "github.com/rclone/rclone/cmd/gendocs"
+	_ "github.com/rclone/rclone/cmd/gitannex"
 	_ "github.com/rclone/rclone/cmd/hashsum"
 	_ "github.com/rclone/rclone/cmd/link"
 	_ "github.com/rclone/rclone/cmd/listremotes"

cmd/gitannex/gitannex.go

@@ -0,0 +1,515 @@
+// Package gitannex provides the "gitannex" command, which enables [git-annex]
+// to communicate with rclone by implementing the [external special remote
+// protocol]. The protocol is line delimited and spoken over stdin and stdout.
+//
+// # Milestones
+//
+// (Tracked in [issue #7625].)
+//
+//  1. ✅ Minimal support for the [external special remote protocol]. Tested on
+//     "local" and "drive" backends.
+//  2. Add support for the ASYNC protocol extension. This may improve performance.
+//  3. Support the [simple export interface]. This will enable `git-annex
+//     export` functionality.
+//  4. Once the draft is finalized, support import/export interface.
+//
+// [git-annex]: https://git-annex.branchable.com/
+// [external special remote protocol]: https://git-annex.branchable.com/design/external_special_remote_protocol/
+// [simple export interface]: https://git-annex.branchable.com/design/external_special_remote_protocol/export_and_import_appendix/
+// [issue #7625]: https://github.com/rclone/rclone/issues/7625
+package gitannex
+
+import (
+	"bufio"
+	"context"
+	_ "embed"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/spf13/cobra"
+)
+
+const subcommandName string = "gitannex"
+const uniqueCommandName string = "git-annex-remote-rclone-builtin"
+
+//go:embed gitannex.md
+var gitannexHelp string
+
+func init() {
+	os.Args = maybeTransformArgs(os.Args)
+	cmd.Root.AddCommand(command)
+}
+
+// maybeTransformArgs returns a modified version of `args` with the "gitannex"
+// subcommand inserted when `args` indicates that the program was executed as
+// "git-annex-remote-rclone-builtin". One way this can happen is when rclone is
+// invoked via symlink. Otherwise, returns `args`.
+func maybeTransformArgs(args []string) []string {
+	if len(args) == 0 || filepath.Base(args[0]) != uniqueCommandName {
+		return args
+	}
+	newArgs := make([]string, 0, len(args)+1)
+	newArgs = append(newArgs, args[0])
+	newArgs = append(newArgs, subcommandName)
+	newArgs = append(newArgs, args[1:]...)
+	return newArgs
+}
+
+// messageParser helps parse messages we receive from git-annex into a sequence
+// of parameters. Messages are not quite trivial to parse because they are
+// separated by spaces, but the final parameter may itself contain spaces.
+//
+// This abstraction is necessary because simply splitting on space doesn't cut
+// it. Also, we cannot know how many parameters to parse until we've parsed the
+// first parameter.
+type messageParser struct {
+	line string
+}
+
+// nextSpaceDelimitedParameter consumes the next space-delimited parameter.
+func (m *messageParser) nextSpaceDelimitedParameter() (string, error) {
+	m.line = strings.TrimRight(m.line, "\r\n")
+	if len(m.line) == 0 {
+		return "", errors.New("nothing remains to parse")
+	}
+
+	before, after, found := strings.Cut(m.line, " ")
+	if found {
+		if len(before) == 0 {
+			return "", fmt.Errorf("found an empty space-delimited parameter in line: %q", m.line)
+		}
+		m.line = after
+		return before, nil
+	}
+
+	remaining := m.line
+	m.line = ""
+	return remaining, nil
+}
+
+// finalParameter consumes the final parameter, which may contain spaces.
+func (m *messageParser) finalParameter() (string, error) {
+	m.line = strings.TrimRight(m.line, "\r\n")
+	if len(m.line) == 0 {
+		return "", errors.New("nothing remains to parse")
+	}
+
+	param := m.line
+	m.line = ""
+	return param, nil
+}
+
+// configDefinition describes a configuration value required by this command. We
+// use "GETCONFIG" messages to query git-annex for these values at runtime.
+type configDefinition struct {
+	name        string
+	description string
+	destination *string
+}
+
+// server contains this command's current state.
+type server struct {
+	reader *bufio.Reader
+	writer io.Writer
+
+	// When true, the server prints a transcript of messages sent and received
+	// to stderr.
+	verbose bool
+
+	extensionInfo                bool
+	extensionAsync               bool
+	extensionGetGitRemoteName    bool
+	extensionUnavailableResponse bool
+
+	configsDone            bool
+	configPrefix           string
+	configRcloneRemoteName string
+}
+
+func (s *server) sendMsg(msg string) {
+	msg = msg + "\n"
+	if _, err := io.WriteString(s.writer, msg); err != nil {
+		panic(err)
+	}
+	if s.verbose {
+		_, err := os.Stderr.WriteString(fmt.Sprintf("server sent %q\n", msg))
+		if err != nil {
+			panic(fmt.Errorf("failed to write verbose message to stderr: %w", err))
+		}
+	}
+}
+
+func (s *server) getMsg() (*messageParser, error) {
+	msg, err := s.reader.ReadString('\n')
+	if err != nil {
+		if len(msg) == 0 {
+			// Git-annex closes stdin when it is done with us, so failing to
+			// read a new line is not an error.
+			return nil, nil
+		}
+		return nil, fmt.Errorf("expected message to end with newline: %q", msg)
+	}
+	if s.verbose {
+		_, err := os.Stderr.WriteString(fmt.Sprintf("server received %q\n", msg))
+		if err != nil {
+			return nil, fmt.Errorf("failed to write verbose message to stderr: %w", err)
+		}
+	}
+	return &messageParser{msg}, nil
+}
+
+func (s *server) run() error {
+	// The remote sends the first message.
+	s.sendMsg("VERSION 2")
+
+	for {
+		message, err := s.getMsg()
+		if err != nil {
+			return fmt.Errorf("error receiving message: %w", err)
+		}
+
+		if message == nil {
+			break
+		}
+
+		command, err := message.nextSpaceDelimitedParameter()
+		if err != nil {
+			return fmt.Errorf("failed to parse command")
+		}
+
+		switch command {
+		//
+		// Git-annex requires that these requests are supported.
+		//
+		case "INITREMOTE":
+			err = s.handleInitRemote()
+		case "PREPARE":
+			err = s.handlePrepare()
+		case "EXPORTSUPPORTED":
+			// Indicate that we do not support exports.
+			s.sendMsg("EXPORTSUPPORTED-FAILURE")
+		case "TRANSFER":
+			err = s.handleTransfer(message)
+		case "CHECKPRESENT":
+			err = s.handleCheckPresent(message)
+		case "REMOVE":
+			err = s.handleRemove(message)
+		case "ERROR":
+			errorMessage, parseErr := message.finalParameter()
+			if parseErr != nil {
+				err = fmt.Errorf("error while parsing ERROR message from git-annex: %w", parseErr)
+				break
+			}
+			err = fmt.Errorf("received error message from git-annex: %s", errorMessage)
+
+		//
+		// These requests are optional.
+		//
+		case "EXTENSIONS":
+			// Git-annex just told us which protocol extensions it supports.
+			// Respond with the list of extensions that we want to use (none).
+			err = s.handleExtensions(message)
+		case "LISTCONFIGS":
+			s.handleListConfigs()
+		case "GETCOST":
+			// Git-annex wants to know the "cost" of using this remote. It
+			// probably depends on the backend we will be using, but let's just
+			// consider this an "expensive remote" per git-annex's
+			// Config/Cost.hs.
+			s.sendMsg("COST 200")
+		case "GETAVAILABILITY":
+			// Indicate that this is a cloud service.
+			s.sendMsg("AVAILABILITY GLOBAL")
+		case "CLAIMURL", "CHECKURL", "WHEREIS", "GETINFO":
+			s.sendMsg("UNSUPPORTED-REQUEST")
+		default:
+			err = fmt.Errorf("received unexpected message from git-annex: %s", message.line)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Idempotently handle an incoming INITREMOTE message. This should perform
+// one-time setup operations, but we may receive the command again, e.g. when
+// this git-annex remote is initialized in a different repository.
+func (s *server) handleInitRemote() error {
+	if err := s.queryConfigs(); err != nil {
+		return fmt.Errorf("failed to get configs: %w", err)
+	}
+
+	remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName))
+	if err != nil {
+		s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote")
+		return fmt.Errorf("failed to open root directory of rclone remote: %w", err)
+	}
+
+	if !remoteRootFs.Features().CanHaveEmptyDirectories {
+		s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories")
+		return fmt.Errorf("rclone remote does not support empty directories")
+	}
+
+	if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil {
+		s.sendMsg("INITREMOTE-FAILURE failed to mkdir")
+		return fmt.Errorf("failed to mkdir: %w", err)
+	}
+
+	s.sendMsg("INITREMOTE-SUCCESS")
+	return nil
+}
+
+// Get a list of configs with pointers to fields of `s`.
+func (s *server) getRequiredConfigs() []configDefinition {
+	return []configDefinition{
+		{
+			"rcloneremotename",
+			"Name of the rclone remote to use. " +
+				"Must match a remote known to rclone. " +
+				"(Note that rclone remotes are a distinct concept from git-annex remotes.)",
+			&s.configRcloneRemoteName,
+		},
+		{
+			"rcloneprefix",
+			"Directory where rclone will write git-annex content. " +
+				"If not specified, defaults to \"git-annex-rclone\". " +
+				"This directory be created on init if it does not exist.",
+			&s.configPrefix,
+		},
+	}
+}
+
+// Query git-annex for config values.
+func (s *server) queryConfigs() error {
+	if s.configsDone {
+		return nil
+	}
+
+	// Send a "GETCONFIG" message for each required config and parse git-annex's
+	// "VALUE" response.
+	for _, config := range s.getRequiredConfigs() {
+		s.sendMsg(fmt.Sprintf("GETCONFIG %s", config.name))
+
+		message, err := s.getMsg()
+		if err != nil {
+			return err
+		}
+
+		valueKeyword, err := message.nextSpaceDelimitedParameter()
+		if err != nil || valueKeyword != "VALUE" {
+			return fmt.Errorf("failed to parse config value: %s %s", valueKeyword, message.line)
+		}
+
+		value, err := message.finalParameter()
+		if err != nil || value == "" {
+			return fmt.Errorf("config value of %q must not be empty", config.name)
+		}
+
+		*config.destination = value
+	}
+
+	s.configsDone = true
+	return nil
+}
+
+func (s *server) handlePrepare() error {
+	if err := s.queryConfigs(); err != nil {
+		s.sendMsg("PREPARE-FAILURE Error getting configs")
+		return fmt.Errorf("error getting configs: %w", err)
+	}
+	s.sendMsg("PREPARE-SUCCESS")
+	return nil
+}
+
+// Git-annex is asking us to return the list of settings that we use. Keep this
+// in sync with `handlePrepare()`.
+func (s *server) handleListConfigs() {
+	for _, config := range s.getRequiredConfigs() {
+		s.sendMsg(fmt.Sprintf("CONFIG %s %s", config.name, config.description))
+	}
+	s.sendMsg("CONFIGEND")
+}
+
+func (s *server) handleTransfer(message *messageParser) error {
+	argMode, err := message.nextSpaceDelimitedParameter()
+	if err != nil {
+		s.sendMsg("TRANSFER-FAILURE failed to parse direction")
+		return fmt.Errorf("malformed arguments for TRANSFER: %w", err)
+	}
+	argKey, err := message.nextSpaceDelimitedParameter()
+	if err != nil {
+		s.sendMsg("TRANSFER-FAILURE failed to parse key")
+		return fmt.Errorf("malformed arguments for TRANSFER: %w", err)
+	}
+	argFile, err := message.finalParameter()
+	if err != nil {
+		s.sendMsg("TRANSFER-FAILURE failed to parse file")
+		return fmt.Errorf("malformed arguments for TRANSFER: %w", err)
+	}
+
+	if err := s.queryConfigs(); err != nil {
+		s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get configs", argMode, argKey))
+		return fmt.Errorf("error getting configs: %w", err)
+	}
+
+	remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+	if err != nil {
+		s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey))
+		return err
+	}
+
+	localDir := filepath.Dir(argFile)
+	localFs, err := cache.Get(context.TODO(), localDir)
+	if err != nil {
+		s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get local fs", argMode, argKey))
+		return fmt.Errorf("failed to get local fs: %w", err)
+	}
+
+	remoteFileName := argKey
+	localFileName := filepath.Base(argFile)
+
+	switch argMode {
+	case "STORE":
+		err = operations.CopyFile(context.TODO(), remoteFs, localFs, remoteFileName, localFileName)
+		if err != nil {
+			s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err))
+			return err
+		}
+
+	case "RETRIEVE":
+		err = operations.CopyFile(context.TODO(), localFs, remoteFs, localFileName, remoteFileName)
+		// It is non-fatal when retrieval fails because the file is missing on
+		// the remote.
+		if err == fs.ErrorObjectNotFound {
+			s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s not found", argMode, argKey))
+			return nil
+		}
+		if err != nil {
+			s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err))
+			return err
+		}
+
+	default:
+		s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s unrecognized mode", argMode, argKey))
+		return fmt.Errorf("received malformed TRANSFER mode: %v", argMode)
+	}
+
+	s.sendMsg(fmt.Sprintf("TRANSFER-SUCCESS %s %s", argMode, argKey))
+	return nil
+}
+
+func (s *server) handleCheckPresent(message *messageParser) error {
+	argKey, err := message.finalParameter()
+	if err != nil {
+		return err
+	}
+
+	if err := s.queryConfigs(); err != nil {
+		s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s failed to get configs", argKey))
+		return fmt.Errorf("error getting configs: %s", err)
+	}
+
+	remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+	if err != nil {
+		s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey))
+		return err
+	}
+
+	_, err = remoteFs.NewObject(context.TODO(), argKey)
+	if err == fs.ErrorObjectNotFound {
+		s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
+		return nil
+	}
+	if err != nil {
+		s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s error finding file", argKey))
+		return err
+	}
+
+	s.sendMsg(fmt.Sprintf("CHECKPRESENT-SUCCESS %s", argKey))
+	return nil
+}
+
+func (s *server) handleRemove(message *messageParser) error {
+	argKey, err := message.finalParameter()
+	if err != nil {
+		return err
+	}
+
+	remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+	if err != nil {
+		s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
+		return fmt.Errorf("error getting remote fs: %w", err)
+	}
+
+	fileObj, err := remoteFs.NewObject(context.TODO(), argKey)
+	// It is non-fatal when removal fails because the file is missing on the
+	// remote.
+	if errors.Is(err, fs.ErrorObjectNotFound) {
+		s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey))
+		return nil
+	}
+	if err != nil {
+		s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error getting new fs object: %s", argKey, err))
+		return fmt.Errorf("error getting new fs object: %w", err)
+	}
+	if err := operations.DeleteFile(context.TODO(), fileObj); err != nil {
+		s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error deleting file", argKey))
+		return fmt.Errorf("error deleting file: %q", argKey)
+	}
+	s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey))
+	return nil
+}
+
+func (s *server) handleExtensions(message *messageParser) error {
+	for {
+		extension, err := message.nextSpaceDelimitedParameter()
+		if err != nil {
+			break
+		}
+		switch extension {
+		case "INFO":
+			s.extensionInfo = true
+		case "ASYNC":
+			s.extensionAsync = true
+		case "GETGITREMOTENAME":
+			s.extensionGetGitRemoteName = true
+		case "UNAVAILABLERESPONSE":
+			s.extensionUnavailableResponse = true
+		}
+	}
+	s.sendMsg("EXTENSIONS")
+	return nil
+}
+
+var command = &cobra.Command{
+	Aliases: []string{uniqueCommandName},
+	Use:     subcommandName,
+	Short:   "Speaks with git-annex over stdin/stdout.",
+	Long:    gitannexHelp,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.67.0",
+	},
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(0, 0, command, args)
+
+		s := server{
+			reader: bufio.NewReader(os.Stdin),
+			writer: os.Stdout,
+		}
+		err := s.run()
+		if err != nil {
+			s.sendMsg(fmt.Sprintf("ERROR %s", err.Error()))
+			panic(err)
+		}
+	},
+}

cmd/gitannex/gitannex.md

@@ -0,0 +1,38 @@
+Rclone's gitannex subcommand enables git-annex to store and retrieve content
+from an rclone remote. It expects to be run by git-annex, not directly by users.
+It is an "external special remote program" as defined by git-annex.
+
+Installation on Linux
+---------------------
+
+1. Create a symlink and ensure it's on your PATH. For example:
+
+        ln -s "$(realpath rclone)" "$HOME/bin/git-annex-remote-rclone-builtin"
+
+2. Add a new external remote to your git-annex repo.
+
+   The new remote's type should be "rclone-builtin". When git-annex interacts
+   with remotes of this type, it will try to run a command named
+   "git-annex-remote-rclone-builtin", so the symlink from the previous step
+   should be on your PATH.
+
+   The following example creates a new git-annex remote named "MyRemote" that
+   will use the rclone remote named "SomeRcloneRemote". This rclone remote must
+   be configured in your rclone.conf file, wherever that is located on your
+   system. The rcloneprefix value ensures that content is only written into the
+   rclone remote underneath the "git-annex-content" directory.
+
+        git annex initremote MyRemote         \
+            type=external                     \
+            externaltype=rclone-builtin       \
+            encryption=none                   \
+            rcloneremotename=SomeRcloneRemote \
+            rcloneprefix=git-annex-content
+
+3. Before you trust this command with your precious data, be sure to **test the
+   remote**. This command is very new and has not been tested on many rclone
+   backends. Caveat emptor!
+
+        git annex testremote my-rclone-remote
+
+Happy annexing!

cmd/gitannex/gitannex_test.go

@@ -0,0 +1,969 @@
+package gitannex
+
+import (
+	"bufio"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	// Without this import, the local filesystem backend would be unavailable.
+	// It looks unused, but the act of importing it runs its `init()` function.
+	_ "github.com/rclone/rclone/backend/local"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configfile"
+	"github.com/rclone/rclone/fstest/mockfs"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFixArgsForSymlinkIdentity(t *testing.T) {
+	for _, argList := range [][]string{
+		[]string{},
+		[]string{"foo"},
+		[]string{"foo", "bar"},
+		[]string{"foo", "bar", "baz"},
+	} {
+		assert.Equal(t, maybeTransformArgs(argList), argList)
+	}
+}
+
+func TestFixArgsForSymlinkCorrectName(t *testing.T) {
+	assert.Equal(t,
+		maybeTransformArgs([]string{"git-annex-remote-rclone-builtin"}),
+		[]string{"git-annex-remote-rclone-builtin", "gitannex"})
+	assert.Equal(t,
+		maybeTransformArgs([]string{"/path/to/git-annex-remote-rclone-builtin"}),
+		[]string{"/path/to/git-annex-remote-rclone-builtin", "gitannex"})
+}
+
+type messageParserTestCase struct {
+	label    string
+	testFunc func(*testing.T)
+}
+
+var messageParserTestCases = []messageParserTestCase{
+	{
+		"OneParam",
+		func(t *testing.T) {
+			m := messageParser{"foo\n"}
+
+			param, err := m.nextSpaceDelimitedParameter()
+			assert.NoError(t, err)
+			assert.Equal(t, param, "foo")
+
+			param, err = m.nextSpaceDelimitedParameter()
+			assert.Error(t, err)
+			assert.Equal(t, param, "")
+
+			param, err = m.finalParameter()
+			assert.Error(t, err)
+			assert.Equal(t, param, "")
+
+			param, err = m.finalParameter()
+			assert.Error(t, err)
+			assert.Equal(t, param, "")
+
+			param, err = m.nextSpaceDelimitedParameter()
+			assert.Error(t, err)
+			assert.Equal(t, param, "")
+
+		},
+	},
+	{
+		"TwoParams",
+		func(t *testing.T) {
+			m := messageParser{"foo bar\n"}
+
+			param, err := m.nextSpaceDelimitedParameter()
+			assert.NoError(t, err)
+			assert.Equal(t, param, "foo")
+
+			param, err = m.nextSpaceDelimitedParameter()
+			assert.NoError(t, err)
+			assert.Equal(t, param, "bar")
+
+			param, err = m.nextSpaceDelimitedParameter()
+			assert.Error(t, err)
+			assert.Equal(t, param, "")
+
+			param, err = m.finalParameter()
+			assert.Error(t, err)
+			assert.Equal(t, param, "")
+		},
+	},
+	{
+		"TwoParamsNoTrailingNewline",
+
+		func(t *testing.T) {
+			m := messageParser{"foo bar"}
+
+			param, err := m.nextSpaceDelimitedParameter()
+			assert.NoError(t, err)
+			assert.Equal(t, param, "foo")
+
+			param, err = m.nextSpaceDelimitedParameter()
+			assert.NoError(t, err)
+			assert.Equal(t, param, "bar")
+
+			param, err = m.nextSpaceDelimitedParameter()
+			assert.Error(t, err)
+			assert.Equal(t, param, "")
+
+			param, err = m.finalParameter()
+			assert.Error(t, err)
+			assert.Equal(t, param, "")
+		},
+	},
+	{
+		"ThreeParamsWhereFinalParamContainsSpaces",
+		func(t *testing.T) {
+			m := messageParser{"firstparam secondparam final param with spaces"}
+
+			param, err := m.nextSpaceDelimitedParameter()
+			assert.NoError(t, err)
+			assert.Equal(t, param, "firstparam")
+
+			param, err = m.nextSpaceDelimitedParameter()
+			assert.NoError(t, err)
+			assert.Equal(t, param, "secondparam")
+
+			param, err = m.finalParameter()
+			assert.NoError(t, err)
+			assert.Equal(t, param, "final param with spaces")
+		},
+	},
+	{
+		"OneLongFinalParameter",
+		func(t *testing.T) {
+			for _, lineEnding := range []string{"", "\n", "\r", "\r\n", "\n\r"} {
+				lineEnding := lineEnding
+				testName := fmt.Sprintf("lineEnding%x", lineEnding)
+
+				t.Run(testName, func(t *testing.T) {
+					m := messageParser{"one long final parameter" + lineEnding}
+
+					param, err := m.finalParameter()
+					assert.NoError(t, err)
+					assert.Equal(t, param, "one long final parameter")
+
+					param, err = m.finalParameter()
+					assert.Error(t, err)
+					assert.Equal(t, param, "")
+				})
+
+			}
+		},
+	},
+	{
+		"MultipleSpaces",
+		func(t *testing.T) {
+			m := messageParser{"foo  bar\n\r"}
+
+			param, err := m.nextSpaceDelimitedParameter()
+			assert.NoError(t, err)
+			assert.Equal(t, param, "foo")
+
+			param, err = m.nextSpaceDelimitedParameter()
+			assert.Error(t, err, "blah")
+			assert.Equal(t, param, "")
+		},
+	},
+	{
+		"StartsWithSpace",
+		func(t *testing.T) {
+			m := messageParser{" foo"}
+
+			param, err := m.nextSpaceDelimitedParameter()
+			assert.Error(t, err, "blah")
+			assert.Equal(t, param, "")
+		},
+	},
+}
+
+func TestMessageParser(t *testing.T) {
+	for _, testCase := range messageParserTestCases {
+		testCase := testCase
+		t.Run(testCase.label, func(t *testing.T) {
+			t.Parallel()
+			testCase.testFunc(t)
+		})
+	}
+}
+
+type testState struct {
+	t                *testing.T
+	server           *server
+	mockStdinW       *io.PipeWriter
+	mockStdoutReader *bufio.Reader
+
+	localFsDir string
+	configPath string
+	remoteName string
+}
+
+func makeTestState(t *testing.T) testState {
+	stdinR, stdinW := io.Pipe()
+	stdoutR, stdoutW := io.Pipe()
+
+	return testState{
+		t: t,
+		server: &server{
+			reader: bufio.NewReader(stdinR),
+			writer: stdoutW,
+		},
+		mockStdinW:       stdinW,
+		mockStdoutReader: bufio.NewReader(stdoutR),
+	}
+}
+
+func (h *testState) requireReadLineExact(line string) {
+	receivedLine, err := h.mockStdoutReader.ReadString('\n')
+	require.NoError(h.t, err)
+	require.Equal(h.t, line+"\n", receivedLine)
+}
+
+func (h *testState) requireWriteLine(line string) {
+	_, err := h.mockStdinW.Write([]byte(line + "\n"))
+	require.NoError(h.t, err)
+}
+
+// Preconfigure the handle. This enables the calling test to skip the PREPARE
+// handshake.
+func (h *testState) preconfigureServer() {
+	h.server.configPrefix = h.localFsDir
+	h.server.configRcloneRemoteName = h.remoteName
+	h.server.configsDone = true
+}
+
+// getUniqueRemoteName returns a valid remote name derived from the given test's
+// name. This is necessary because when a test registers a second remote with
+// the same name, the original remote appears to take precedence. This function
+// is injective, so each test gets a unique remote name. Returned strings
+// contain no spaces.
+func getUniqueRemoteName(t *testing.T) string {
+	// Using sha256 as a hack to ensure injectivity without adding a global
+	// variable.
+	return fmt.Sprintf("remote-%x", sha256.Sum256([]byte(t.Name())))
+}
+
+type testCase struct {
+	label            string
+	testProtocolFunc func(*testing.T, *testState)
+	expectedError    string
+}
+
+// These test cases run against the "local" backend.
+var localBackendTestCases = []testCase{
+	{
+		label: "HandlesInit",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "HandlesPrepare",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
+			h.requireReadLineExact("EXTENSIONS")
+
+			if !h.server.extensionInfo {
+				t.Errorf("expected INFO extension to be enabled")
+				return
+			}
+
+			h.requireWriteLine("PREPARE")
+			h.requireReadLineExact("GETCONFIG rcloneremotename")
+			h.requireWriteLine("VALUE " + h.remoteName)
+			h.requireReadLineExact("GETCONFIG rcloneprefix")
+			h.requireWriteLine("VALUE " + h.localFsDir)
+			h.requireReadLineExact("PREPARE-SUCCESS")
+
+			require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
+			require.Equal(t, h.server.configPrefix, h.localFsDir)
+			require.True(t, h.server.configsDone)
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "HandlesPrepareAndDoesNotTrimWhitespaceFromValue",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
+			h.requireReadLineExact("EXTENSIONS")
+
+			if !h.server.extensionInfo {
+				t.Errorf("expected INFO extension to be enabled")
+				return
+			}
+
+			h.requireWriteLine("PREPARE")
+			h.requireReadLineExact("GETCONFIG rcloneremotename")
+
+			remoteNameWithSpaces := fmt.Sprintf(" %s ", h.remoteName)
+			localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)
+
+			h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))
+			h.requireReadLineExact("GETCONFIG rcloneprefix")
+
+			h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))
+			h.requireReadLineExact("PREPARE-SUCCESS")
+
+			require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)
+			require.Equal(t, h.server.configPrefix, localFsDirWithSpaces)
+			require.True(t, h.server.configsDone)
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "HandlesEarlyError",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("ERROR foo")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+		expectedError: "received error message from git-annex: foo",
+	},
+	// Test what happens when the git-annex client sends "GETCONFIG", but
+	// doesn't understand git-annex's response.
+	{
+		label: "ConfigFail",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
+			h.requireReadLineExact("EXTENSIONS")
+			require.True(t, h.server.extensionInfo)
+
+			h.requireWriteLine("PREPARE")
+			h.requireReadLineExact("GETCONFIG rcloneremotename")
+			h.requireWriteLine("ERROR ineffable error")
+			h.requireReadLineExact("PREPARE-FAILURE Error getting configs")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+		expectedError: "failed to parse config value: ERROR ineffable error",
+	},
+	{
+		label: "TransferStoreEmptyPath",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			// Note the whitespace following the key.
+			h.requireWriteLine("TRANSFER STORE Key ")
+			h.requireReadLineExact("TRANSFER-FAILURE failed to parse file")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+		expectedError: "malformed arguments for TRANSFER: nothing remains to parse",
+	},
+	// Repeated EXTENSIONS messages add to each other rather than overriding
+	// prior advertised extensions. This behavior is not mandated by the
+	// protocol design.
+	{
+		label: "ExtensionsCompound",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			h.requireWriteLine("EXTENSIONS")
+			h.requireReadLineExact("EXTENSIONS")
+			require.False(t, h.server.extensionInfo)
+			require.False(t, h.server.extensionAsync)
+			require.False(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			h.requireWriteLine("EXTENSIONS INFO")
+			h.requireReadLineExact("EXTENSIONS")
+			require.True(t, h.server.extensionInfo)
+			require.False(t, h.server.extensionAsync)
+			require.False(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			h.requireWriteLine("EXTENSIONS ASYNC")
+			h.requireReadLineExact("EXTENSIONS")
+			require.True(t, h.server.extensionInfo)
+			require.True(t, h.server.extensionAsync)
+			require.False(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			h.requireWriteLine("EXTENSIONS GETGITREMOTENAME")
+			h.requireReadLineExact("EXTENSIONS")
+			require.True(t, h.server.extensionInfo)
+			require.True(t, h.server.extensionAsync)
+			require.True(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			h.requireWriteLine("EXTENSIONS UNAVAILABLERESPONSE")
+			h.requireReadLineExact("EXTENSIONS")
+			require.True(t, h.server.extensionInfo)
+			require.True(t, h.server.extensionAsync)
+			require.True(t, h.server.extensionGetGitRemoteName)
+			require.True(t, h.server.extensionUnavailableResponse)
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "ExtensionsIdempotent",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			h.requireWriteLine("EXTENSIONS")
+			h.requireReadLineExact("EXTENSIONS")
+			require.False(t, h.server.extensionInfo)
+			require.False(t, h.server.extensionAsync)
+			require.False(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			h.requireWriteLine("EXTENSIONS")
+			h.requireReadLineExact("EXTENSIONS")
+			require.False(t, h.server.extensionInfo)
+			require.False(t, h.server.extensionAsync)
+			require.False(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			h.requireWriteLine("EXTENSIONS INFO")
+			h.requireReadLineExact("EXTENSIONS")
+			require.True(t, h.server.extensionInfo)
+			require.False(t, h.server.extensionAsync)
+			require.False(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			h.requireWriteLine("EXTENSIONS INFO")
+			h.requireReadLineExact("EXTENSIONS")
+			require.True(t, h.server.extensionInfo)
+			require.False(t, h.server.extensionAsync)
+			require.False(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			h.requireWriteLine("EXTENSIONS ASYNC ASYNC")
+			h.requireReadLineExact("EXTENSIONS")
+			require.True(t, h.server.extensionInfo)
+			require.True(t, h.server.extensionAsync)
+			require.False(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "ExtensionsSupportsMultiple",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			h.requireWriteLine("EXTENSIONS")
+			h.requireReadLineExact("EXTENSIONS")
+			require.False(t, h.server.extensionInfo)
+			require.False(t, h.server.extensionAsync)
+			require.False(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			h.requireWriteLine("EXTENSIONS INFO ASYNC")
+			h.requireReadLineExact("EXTENSIONS")
+			require.True(t, h.server.extensionInfo)
+			require.True(t, h.server.extensionAsync)
+			require.False(t, h.server.extensionGetGitRemoteName)
+			require.False(t, h.server.extensionUnavailableResponse)
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "TransferStoreAbsolute",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			// Create temp file for transfer with an absolute path.
+			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+			require.FileExists(t, fileToTransfer)
+			require.True(t, filepath.IsAbs(fileToTransfer))
+
+			// Specify an absolute path to transfer.
+			h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
+			h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
+			require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
+
+			// Transfer the same absolute path a second time, but with a different key.
+			h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + fileToTransfer)
+			h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute2")
+			require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute2"))
+
+			h.requireWriteLine("CHECKPRESENT KeyAbsolute2")
+			h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyAbsolute2")
+
+			h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
+			h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	// Test that the TRANSFER command understands simple relative paths
+	// consisting only of a file name.
+	{
+		label: "TransferStoreRelative",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			// Save the current working directory so we can restore it when this
+			// test ends.
+			cwd, err := os.Getwd()
+			require.NoError(t, err)
+
+			require.NoError(t, os.Chdir(t.TempDir()))
+			t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			// Create temp file for transfer with a relative path.
+			fileToTransfer := "file.txt"
+			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+			require.FileExists(t, fileToTransfer)
+			require.False(t, filepath.IsAbs(fileToTransfer))
+
+			// Specify a relative path to transfer.
+			h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
+			h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
+			require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
+
+			h.requireWriteLine("CHECKPRESENT KeyRelative")
+			h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
+
+			h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
+			h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "TransferStorePathWithInteriorWhitespace",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			// Save the current working directory so we can restore it when this
+			// test ends.
+			cwd, err := os.Getwd()
+			require.NoError(t, err)
+
+			require.NoError(t, os.Chdir(t.TempDir()))
+			t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
+
+			h.preconfigureServer()
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			// Create temp file for transfer.
+			fileToTransfer := "filename with spaces.txt"
+			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+			require.FileExists(t, fileToTransfer)
+			require.False(t, filepath.IsAbs(fileToTransfer))
+
+			// Specify a relative path to transfer.
+			h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
+			h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
+			require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
+
+			h.requireWriteLine("CHECKPRESENT KeyRelative")
+			h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
+
+			h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
+			h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "CheckPresentAndTransfer",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			// Create temp file for transfer.
+			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
+			h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
+
+			// Specify an absolute path to transfer.
+			require.True(t, filepath.IsAbs(fileToTransfer))
+			h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
+			h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
+			require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	// Check whether a key is present, transfer a file with that key, then check
+	// again whether it is present.
+	//
+	// This is a regression test for a bug where the second CHECKPRESENT would
+	// generate the following response:
+	//
+	//	CHECKPRESENT-UNKNOWN ${key} failed to read directory entry: readdirent ${filepath}: not a directory
+	//
+	// This message was generated by the local backend's `List()` function. When
+	// checking whether a file exists, we were erroneously listing its contents as
+	// if it were a directory.
+	{
+		label: "CheckpresentTransferCheckpresent",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			// Create temp file for transfer.
+			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			h.requireWriteLine("CHECKPRESENT foo")
+			h.requireReadLineExact("CHECKPRESENT-FAILURE foo")
+
+			h.requireWriteLine("TRANSFER STORE foo " + fileToTransfer)
+			h.requireReadLineExact("TRANSFER-SUCCESS STORE foo")
+			require.FileExists(t, filepath.Join(h.localFsDir, "foo"))
+
+			h.requireWriteLine("CHECKPRESENT foo")
+			h.requireReadLineExact("CHECKPRESENT-SUCCESS foo")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "TransferAndCheckpresentWithRealisticKey",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			// Create temp file for transfer.
+			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			realisticKey := "SHA256E-s1048576--7ba87e06b9b7903cfbaf4a38736766c161e3e7b42f06fe57f040aa410a8f0701.this-is-a-test-key"
+
+			// Specify an absolute path to transfer.
+			require.True(t, filepath.IsAbs(fileToTransfer))
+			h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, fileToTransfer))
+			h.requireReadLineExact("TRANSFER-SUCCESS STORE " + realisticKey)
+			require.FileExists(t, filepath.Join(h.localFsDir, realisticKey))
+
+			h.requireWriteLine("CHECKPRESENT " + realisticKey)
+			h.requireReadLineExact("CHECKPRESENT-SUCCESS " + realisticKey)
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "RetrieveNonexistentFile",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			h.requireWriteLine("TRANSFER RETRIEVE SomeKey path")
+			h.requireReadLineExact("TRANSFER-FAILURE RETRIEVE SomeKey not found")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "StoreCheckpresentRetrieve",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			// Create temp file for transfer.
+			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			// Specify an absolute path to transfer.
+			require.True(t, filepath.IsAbs(fileToTransfer))
+			h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
+			h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
+			require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+			h.requireWriteLine("CHECKPRESENT SomeKey")
+			h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
+
+			retrievedFilePath := fileToTransfer + ".retrieved"
+			require.NoFileExists(t, retrievedFilePath)
+			h.requireWriteLine("TRANSFER RETRIEVE SomeKey " + retrievedFilePath)
+			h.requireReadLineExact("TRANSFER-SUCCESS RETRIEVE SomeKey")
+			require.FileExists(t, retrievedFilePath)
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "RemovePreexistingFile",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			// Write a file into the remote without using the git-annex
+			// protocol.
+			remoteFilePath := filepath.Join(h.localFsDir, "SomeKey")
+			require.NoError(t, os.WriteFile(remoteFilePath, []byte("HELLO"), 0600))
+			require.FileExists(t, remoteFilePath)
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			h.requireWriteLine("CHECKPRESENT SomeKey")
+			h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
+			require.FileExists(t, remoteFilePath)
+
+			h.requireWriteLine("REMOVE SomeKey")
+			h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
+			require.NoFileExists(t, remoteFilePath)
+
+			h.requireWriteLine("CHECKPRESENT SomeKey")
+			h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
+			require.NoFileExists(t, remoteFilePath)
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "Remove",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			// Create temp file for transfer.
+			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			h.requireWriteLine("CHECKPRESENT SomeKey")
+			h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
+
+			// Specify an absolute path to transfer.
+			require.True(t, filepath.IsAbs(fileToTransfer))
+			h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
+			h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
+			require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+			h.requireWriteLine("CHECKPRESENT SomeKey")
+			h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
+
+			h.requireWriteLine("REMOVE SomeKey")
+			h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
+			require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+			h.requireWriteLine("CHECKPRESENT SomeKey")
+			h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "RemoveNonexistentFile",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			// Create temp file for transfer.
+			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			h.requireWriteLine("CHECKPRESENT SomeKey")
+			h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
+
+			require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+			h.requireWriteLine("REMOVE SomeKey")
+			h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
+			require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+			h.requireWriteLine("CHECKPRESENT SomeKey")
+			h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+	{
+		label: "ExportNotSupported",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.preconfigureServer()
+
+			h.requireReadLineExact("VERSION 2")
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+			h.requireWriteLine("EXPORTSUPPORTED")
+			h.requireReadLineExact("EXPORTSUPPORTED-FAILURE")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+	},
+}
+
+func TestGitAnnexLocalBackendCases(t *testing.T) {
+	for _, testCase := range localBackendTestCases {
+		// Clear global state left behind by tests that chdir to a temp directory.
+		cache.Clear()
+
+		// TODO: Remove this when rclone requires a Go version >= 1.22. Future
+		// versions of Go fix the semantics of capturing a range variable.
+		// https://go.dev/blog/loopvar-preview
+		testCase := testCase
+
+		t.Run(testCase.label, func(t *testing.T) {
+			tempDir := t.TempDir()
+
+			// Create temp dir for an rclone remote pointing at local filesystem.
+			localFsDir := filepath.Join(tempDir, "remoteTarget")
+			require.NoError(t, os.Mkdir(localFsDir, 0700))
+
+			// Create temp config
+			remoteName := getUniqueRemoteName(t)
+			configLines := []string{
+				fmt.Sprintf("[%s]", remoteName),
+				"type = local",
+				fmt.Sprintf("remote = %s", localFsDir),
+			}
+			configContents := strings.Join(configLines, "\n")
+
+			configPath := filepath.Join(tempDir, "rclone.conf")
+			require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
+			require.NoError(t, config.SetConfigPath(configPath))
+
+			// The custom config file will be ignored unless we install the
+			// global config file handler.
+			configfile.Install()
+
+			handle := makeTestState(t)
+			handle.localFsDir = localFsDir
+			handle.configPath = configPath
+			handle.remoteName = remoteName
+
+			var wg sync.WaitGroup
+			wg.Add(1)
+
+			go func() {
+				err := handle.server.run()
+
+				if testCase.expectedError == "" {
+					require.NoError(t, err)
+				} else {
+					require.ErrorContains(t, err, testCase.expectedError)
+				}
+
+				wg.Done()
+			}()
+			defer wg.Wait()
+
+			testCase.testProtocolFunc(t, &handle)
+		})
+	}
+}
+
+// Configure the git-annex client with a mockfs backend and send it the
+// "INITREMOTE" command over mocked stdin. This should fail because mockfs does
+// not support empty directories.
+func TestGitAnnexHandleInitRemoteBackendDoesNotSupportEmptyDirectories(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Temporarily override the filesystem registry.
+	oldRegistry := fs.Registry
+	mockfs.Register()
+	defer func() { fs.Registry = oldRegistry }()
+
+	// Create temp dir for an rclone remote pointing at local filesystem.
+	localFsDir := filepath.Join(tempDir, "remoteTarget")
+	require.NoError(t, os.Mkdir(localFsDir, 0700))
+
+	// Create temp config
+	remoteName := getUniqueRemoteName(t)
+	configLines := []string{
+		fmt.Sprintf("[%s]", remoteName),
+		"type = mockfs",
+		fmt.Sprintf("remote = %s", localFsDir),
+	}
+	configContents := strings.Join(configLines, "\n")
+
+	configPath := filepath.Join(tempDir, "rclone.conf")
+	require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
+
+	// The custom config file will be ignored unless we install the global
+	// config file handler.
+	configfile.Install()
+	require.NoError(t, config.SetConfigPath(configPath))
+
+	handle := makeTestState(t)
+	handle.server.configPrefix = localFsDir
+	handle.server.configRcloneRemoteName = remoteName
+	handle.server.configsDone = true
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	go func() {
+		require.NotNil(t, handle.server.run())
+		wg.Done()
+	}()
+	defer wg.Wait()
+
+	handle.requireReadLineExact("VERSION 2")
+	handle.requireWriteLine("INITREMOTE")
+	handle.requireReadLineExact("INITREMOTE-FAILURE this rclone remote does not support empty directories")
+}

cmd/gitannex/run-git-annex-testremote.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+#
+# End-to-end tests for "rclone gitannex". This script runs the `git-annex
+# testremote` suite against "rclone gitannex" in an ephemeral git-annex repo.
+#
+# Assumptions:
+#
+#   * This system has an rclone remote configured named "git-annex-builtin-test-remote".
+#
+#   * If it uses rclone's "local" backend, /tmp/git-annex-builtin-test-remote exists.
+
+set -e
+
+TEST_DIR="$(realpath "$(mktemp -d)")"
+mkdir "$TEST_DIR/bin"
+
+function cleanup()
+{
+    rm -rf "$TEST_DIR"
+}
+
+trap cleanup EXIT
+
+RCLONE_DIR="$(git rev-parse --show-toplevel)"
+
+rm -rf /tmp/git-annex-builtin-test-remote/*
+
+set -x
+
+pushd "$RCLONE_DIR"
+go build -o "$TEST_DIR/bin" ./
+
+ln -s "$(realpath "$TEST_DIR/bin/rclone")" "$TEST_DIR/bin/git-annex-remote-rclone-builtin"
+popd
+
+pushd "$TEST_DIR"
+
+git init
+git annex init
+
+REMOTE_NAME=git-annex-builtin-test-remote
+PREFIX=/tmp/git-annex-builtin-test-remote
+
+PATH="$PATH:$TEST_DIR/bin" git annex initremote $REMOTE_NAME \
+    type=external externaltype=rclone-builtin encryption=none \
+    rcloneremotename=$REMOTE_NAME \
+    rcloneprefix="$PREFIX"
+
+PATH="$PATH:$(realpath bin)" git annex testremote $REMOTE_NAME
+
+popd
+rm -rf "$TEST_DIR"