From ba11040d6b5363789535d8693a1e675301e8c32e Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Tue, 21 Nov 2023 10:30:47 +0000
Subject: [PATCH] s3: detect looping when using gcs and versions

Apparently gcs doesn't return an S3 compatible result when using
versions.

In particular it doesn't return a NextKeyMarker - this means rclone
loops and fetches the same page over and over again.

This patch detects the problem and stops the infinite retries but it
doesn't fix the underlying problem.

See: https://forum.rclone.org/t/list-s3-versions-files-looping-bug/42974
See: https://issuetracker.google.com/u/0/issues/312292516
---
 backend/s3/s3.go   | 3 +++
 docs/content/s3.md | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index d4f686735..bb4e97c61 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -3721,6 +3721,9 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output,
 	// Set up the request for next time
 	ls.req.KeyMarker = respVersions.NextKeyMarker
 	ls.req.VersionIdMarker = respVersions.NextVersionIdMarker
+	if aws.BoolValue(respVersions.IsTruncated) && ls.req.KeyMarker == nil {
+		return nil, nil, errors.New("s3 protocol error: received versions listing with IsTruncated set with no NextKeyMarker")
+	}
 
 	// If we are URL encoding then must decode the marker
 	if ls.req.KeyMarker != nil && ls.req.EncodingType != nil {
diff --git a/docs/content/s3.md b/docs/content/s3.md
index b1853b3e6..b4bdf10f8 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -3685,6 +3685,12 @@ secret_access_key = your_secret_key
 endpoint = https://storage.googleapis.com
 ```
 
+**Note** that `--s3-versions` does not work with GCS when it needs to do directory paging. Rclone will return the error:
+
+    s3 protocol error: received versions listing with IsTruncated set with no NextKeyMarker
+
+This is Google bug [#312292516](https://issuetracker.google.com/u/0/issues/312292516).
+
 ### DigitalOcean Spaces
 
 [Spaces](https://www.digitalocean.com/products/object-storage/) is an [S3-interoperable](https://developers.digitalocean.com/documentation/spaces/) object storage service from cloud provider DigitalOcean.