s3: make v2 signatures work for ceph

s3/s3.go

@@ -11,18 +11,12 @@ Progress of port to aws-sdk
 What happens if you CTRL-C a multipart upload
   * get an incomplete upload
   * disappears when you delete the bucket
-
-Doesn't support v2 signing so can't interface with Ceph
-  * http://tracker.ceph.com/issues/10333
-  * https://github.com/aws/aws-sdk-go/issues/291
-
 */
 
 import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"path"
 	"regexp"
 	"strings"
@@ -31,6 +25,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/service"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/ncw/rclone/fs"
@@ -205,11 +200,27 @@ func s3Connection(name string) (*s3.S3, error) {
 		WithMaxRetries(maxRetries).
 		WithCredentials(auth).
 		WithEndpoint(endpoint).
-		WithHTTPClient(fs.Config.Client())
+		WithHTTPClient(fs.Config.Client()).
+		WithS3ForcePathStyle(true)
+	// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
 	c := s3.New(awsConfig)
 	if region == "other-v2-signature" {
-		log.Fatal("Sorry v2 signatures not supported yet :-(")
+		fs.Debug(name, "Using v2 auth")
+		signer := func(req *service.Request) {
+			// Ignore AnonymousCredentials object
+			if req.Service.Config.Credentials == credentials.AnonymousCredentials {
+				return
+			}
+			sign(accessKeyId, secretAccessKey, req.HTTPRequest)
+		}
+		c.Handlers.Sign.Clear()
+		c.Handlers.Sign.PushBack(service.BuildContentLength)
+		c.Handlers.Sign.PushBack(signer)
 	}
+	// Add user agent
+	c.Handlers.Build.PushBack(func(r *service.Request) {
+		r.HTTPRequest.Header.Set("User-Agent", fs.UserAgent)
+	})
 	return c, nil
 }
 

s3/v2sign.go

@@ -0,0 +1,115 @@
+// v2 signing
+
+package s3
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+)
+
+// URL parameters that need to be added to the signature
+var s3ParamsToSign = map[string]struct{}{
+	"acl":                          struct{}{},
+	"location":                     struct{}{},
+	"logging":                      struct{}{},
+	"notification":                 struct{}{},
+	"partNumber":                   struct{}{},
+	"policy":                       struct{}{},
+	"requestPayment":               struct{}{},
+	"torrent":                      struct{}{},
+	"uploadId":                     struct{}{},
+	"uploads":                      struct{}{},
+	"versionId":                    struct{}{},
+	"versioning":                   struct{}{},
+	"versions":                     struct{}{},
+	"response-content-type":        struct{}{},
+	"response-content-language":    struct{}{},
+	"response-expires":             struct{}{},
+	"response-cache-control":       struct{}{},
+	"response-content-disposition": struct{}{},
+	"response-content-encoding":    struct{}{},
+}
+
+// sign signs requests using v2 auth
+//
+// Cobbled together from goamz and aws-sdk-go
+func sign(AccessKey, SecretKey string, req *http.Request) {
+	// Set date
+	date := time.Now().UTC().Format(time.RFC1123)
+	req.Header.Set("Date", date)
+
+	// Sort out URI
+	uri := req.URL.Opaque
+	if uri != "" {
+		if strings.HasPrefix(uri, "//") {
+			// Strip off //host/uri
+			uri = "/" + strings.Join(strings.Split(uri, "/")[3:], "/")
+			req.URL.Opaque = uri // reset to plain URI otherwise Ceph gets confused
+		}
+	} else {
+		uri = req.URL.Path
+	}
+	if uri == "" {
+		uri = "/"
+	}
+
+	// Look through headers of interest
+	var md5 string
+	var contentType string
+	var headersToSign []string
+	for k, v := range req.Header {
+		k = strings.ToLower(k)
+		switch k {
+		case "content-md5":
+			md5 = v[0]
+		case "content-type":
+			contentType = v[0]
+		default:
+			if strings.HasPrefix(k, "x-amz-") {
+				vall := strings.Join(v, ",")
+				headersToSign = append(headersToSign, k+":"+vall)
+			}
+		}
+	}
+	// Make headers of interest into canonical string
+	var joinedHeadersToSign string
+	if len(headersToSign) > 0 {
+		sort.StringSlice(headersToSign).Sort()
+		joinedHeadersToSign = strings.Join(headersToSign, "\n") + "\n"
+	}
+
+	// Look for query parameters which need to be added to the signature
+	params := req.URL.Query()
+	var queriesToSign []string
+	for k, vs := range params {
+		if _, ok := s3ParamsToSign[k]; ok {
+			for _, v := range vs {
+				if v == "" {
+					queriesToSign = append(queriesToSign, k)
+				} else {
+					queriesToSign = append(queriesToSign, k+"="+v)
+				}
+			}
+		}
+	}
+	// Add query parameters to URI
+	if len(queriesToSign) > 0 {
+		sort.StringSlice(queriesToSign).Sort()
+		uri += "?" + strings.Join(queriesToSign, "&")
+	}
+
+	// Make signature
+	payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri
+	hash := hmac.New(sha1.New, []byte(SecretKey))
+	hash.Write([]byte(payload))
+	signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size()))
+	base64.StdEncoding.Encode(signature, hash.Sum(nil))
+
+	// Set signature in request
+	req.Header.Set("Authorization", "AWS "+AccessKey+":"+string(signature))
+}