swift: obey Retry-After to fix OVH restore from cold storage

In as many methods as possible we attempt to obey the Retry-After
header where it is provided.

This means that when objects are being requested from OVH cold storage
rclone will sleep the correct amount of time before retrying.

If the sleeps are short it does them immediately, if long then it
returns an ErrorRetryAfter which will cause the outer retry to sleep
before retrying.

Fixes #3041

backend/swift/swift_internal_test.go

@@ -1,6 +1,13 @@
 package swift
 
-import "testing"
+import (
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/ncw/swift"
+	"github.com/stretchr/testify/assert"
+)
 
 func TestInternalUrlEncode(t *testing.T) {
 	for _, test := range []struct {
@@ -23,3 +30,37 @@ func TestInternalUrlEncode(t *testing.T) {
 		}
 	}
 }
+
+func TestInternalShouldRetryHeaders(t *testing.T) {
+	headers := swift.Headers{
+		"Content-Length": "64",
+		"Content-Type":   "text/html; charset=UTF-8",
+		"Date":           "Mon: 18 Mar 2019 12:11:23 GMT",
+		"Retry-After":    "1",
+	}
+	err := &swift.Error{
+		StatusCode: 429,
+		Text:       "Too Many Requests",
+	}
+
+	// Short sleep should just do the sleep
+	start := time.Now()
+	retry, gotErr := shouldRetryHeaders(headers, err)
+	dt := time.Since(start)
+	assert.True(t, retry)
+	assert.Equal(t, err, gotErr)
+	assert.True(t, dt > time.Second/2)
+
+	// Long sleep should return RetryError
+	headers["Retry-After"] = "3600"
+	start = time.Now()
+	retry, gotErr = shouldRetryHeaders(headers, err)
+	dt = time.Since(start)
+	assert.True(t, dt < time.Second)
+	assert.False(t, retry)
+	assert.Equal(t, true, fserrors.IsRetryAfterError(gotErr))
+	after := gotErr.(fserrors.RetryAfter).RetryAfter()
+	dt = after.Sub(start)
+	assert.True(t, dt >= time.Hour-time.Second && dt <= time.Hour+time.Second)
+
+}