s3: fix v2 auth for multipart server side copy

Before this change the v2 signer sorted the headers for signing as
joined key:value pairs. However this put these two headers in the
wrong order.

    x-amz-copy-source-range:
    x-amz-copy-source:

This changes sorts on the keys before joining the values producing the
correct sort order.

    x-amz-copy-source:
    x-amz-copy-source-range:

This commit also adds some missing query parameters for signing that I
spotted in the s3cmd source.

backend/s3/s3.go

@@ -1523,7 +1523,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 			if req.Config.Credentials == credentials.AnonymousCredentials {
 				return
 			}
-			sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
+			v2sign(opt, req.HTTPRequest)
 		}
 		c.Handlers.Sign.Clear()
 		c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
@@ -3202,6 +3202,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// Read the metadata from the newly created object
 	o.meta = nil // wipe old metadata
 	err = o.readMetaData(ctx)
+	// Empty an Etag which is a valid md5sum for multipart
+	// uploads. This works around a bug in KS3 where the ETag is a
+	// correctly formed md5sum for multpart uploads
+	if multipart && matchMd5.MatchString(strings.Trim(strings.ToLower(o.etag), `"`)) {
+		o.etag = ""
+	}
 	return err
 }
 

backend/s3/v2sign.go

@@ -9,7 +9,10 @@ import (
 	"net/http"
 	"sort"
 	"strings"
+	"sync"
 	"time"
+
+	"github.com/rclone/rclone/fs"
 )
 
 // URL parameters that need to be added to the signature
@@ -33,12 +36,20 @@ var s3ParamsToSign = map[string]struct{}{
 	"response-cache-control":       {},
 	"response-content-disposition": {},
 	"response-content-encoding":    {},
+	"lifecycle":                    {},
+	"website":                      {},
+	"delete":                       {},
+	"cors":                         {},
+	"restore":                      {},
 }
 
+// Warn once about empty endpoint
+var warnEmptyEndpointOnce sync.Once
+
 // sign signs requests using v2 auth
 //
 // Cobbled together from goamz and aws-sdk-go
-func sign(AccessKey, SecretKey string, req *http.Request) {
+func v2sign(opt *Options, req *http.Request) {
 	// Set date
 	date := time.Now().UTC().Format(time.RFC1123)
 	req.Header.Set("Date", date)
@@ -48,11 +59,26 @@ func sign(AccessKey, SecretKey string, req *http.Request) {
 	if uri == "" {
 		uri = "/"
 	}
+	// If not using path style then need to stick the bucket on
+	// the start of the requests if doing a bucket based query
+	if !opt.ForcePathStyle {
+		if opt.Endpoint == "" {
+			warnEmptyEndpointOnce.Do(func() {
+				fs.Logf(nil, "If using v2 auth with AWS and force_path_style=false, endpoint must be set in the config")
+			})
+		} else if req.URL.Host != opt.Endpoint {
+			// read the bucket off the start of the hostname
+			i := strings.IndexRune(req.URL.Host, '.')
+			if i >= 0 {
+				uri = "/" + req.URL.Host[:i] + uri
+			}
+		}
+	}
 
 	// Look through headers of interest
 	var md5 string
 	var contentType string
-	var headersToSign []string
+	var headersToSign [][2]string // slice of key, value pairs
 	for k, v := range req.Header {
 		k = strings.ToLower(k)
 		switch k {
@@ -63,15 +89,26 @@ func sign(AccessKey, SecretKey string, req *http.Request) {
 		default:
 			if strings.HasPrefix(k, "x-amz-") {
 				vall := strings.Join(v, ",")
-				headersToSign = append(headersToSign, k+":"+vall)
+				headersToSign = append(headersToSign, [2]string{k, vall})
 			}
 		}
 	}
 	// Make headers of interest into canonical string
 	var joinedHeadersToSign string
 	if len(headersToSign) > 0 {
-		sort.StringSlice(headersToSign).Sort()
-		joinedHeadersToSign = strings.Join(headersToSign, "\n") + "\n"
+		// sort by keys
+		sort.Slice(headersToSign, func(i, j int) bool {
+			return headersToSign[i][0] < headersToSign[j][0]
+		})
+		// join into key:value\n
+		var out strings.Builder
+		for _, kv := range headersToSign {
+			out.WriteString(kv[0])
+			out.WriteRune(':')
+			out.WriteString(kv[1])
+			out.WriteRune('\n')
+		}
+		joinedHeadersToSign = out.String()
 	}
 
 	// Look for query parameters which need to be added to the signature
@@ -96,11 +133,11 @@ func sign(AccessKey, SecretKey string, req *http.Request) {
 
 	// Make signature
 	payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri
-	hash := hmac.New(sha1.New, []byte(SecretKey))
+	hash := hmac.New(sha1.New, []byte(opt.SecretAccessKey))
 	_, _ = hash.Write([]byte(payload))
 	signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size()))
 	base64.StdEncoding.Encode(signature, hash.Sum(nil))
 
 	// Set signature in request
-	req.Header.Set("Authorization", "AWS "+AccessKey+":"+string(signature))
+	req.Header.Set("Authorization", "AWS "+opt.AccessKeyID+":"+string(signature))
 }