2014-12-22 22:24:45 +00:00
|
|
|
// Package s3 provides a storagedriver.StorageDriver implementation to
|
|
|
|
// store blobs in Amazon S3 cloud storage.
|
|
|
|
//
|
2016-01-22 02:17:53 +00:00
|
|
|
// This package leverages the official aws client library for interfacing with
|
|
|
|
// S3.
|
2014-12-22 22:24:45 +00:00
|
|
|
//
|
2016-01-22 02:17:53 +00:00
|
|
|
// Because S3 is a key, value store the Stat call does not support last modification
|
2014-12-22 22:24:45 +00:00
|
|
|
// time for directories (directories are an abstraction for key, value stores)
|
|
|
|
//
|
2016-01-22 02:17:53 +00:00
|
|
|
// Keep in mind that S3 guarantees only read-after-write consistency for new
|
|
|
|
// objects, but no read-after-update or list-after-write consistency.
|
2014-10-24 23:37:25 +00:00
|
|
|
package s3
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
2017-08-11 22:31:16 +00:00
|
|
|
"context"
|
2017-03-03 19:53:55 +00:00
|
|
|
"crypto/tls"
|
2022-04-23 11:37:09 +00:00
|
|
|
"errors"
|
2014-10-29 01:15:40 +00:00
|
|
|
"fmt"
|
2014-10-24 23:37:25 +00:00
|
|
|
"io"
|
2016-08-16 00:12:24 +00:00
|
|
|
"math"
|
2014-10-24 23:37:25 +00:00
|
|
|
"net/http"
|
2021-06-24 18:42:02 +00:00
|
|
|
"path/filepath"
|
2015-04-22 21:31:34 +00:00
|
|
|
"reflect"
|
2016-04-23 10:13:15 +00:00
|
|
|
"sort"
|
2014-10-24 23:37:25 +00:00
|
|
|
"strconv"
|
2014-12-19 17:16:51 +00:00
|
|
|
"strings"
|
2023-09-22 13:53:47 +00:00
|
|
|
"sync"
|
2014-12-19 17:16:51 +00:00
|
|
|
"time"
|
2014-10-24 23:37:25 +00:00
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
|
|
"github.com/aws/aws-sdk-go/aws/awserr"
|
|
|
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
2018-08-20 15:22:09 +00:00
|
|
|
"github.com/aws/aws-sdk-go/aws/endpoints"
|
2016-01-22 02:17:53 +00:00
|
|
|
"github.com/aws/aws-sdk-go/aws/request"
|
|
|
|
"github.com/aws/aws-sdk-go/aws/session"
|
|
|
|
"github.com/aws/aws-sdk-go/service/s3"
|
2015-04-27 22:58:58 +00:00
|
|
|
|
2020-08-24 11:18:39 +00:00
|
|
|
dcontext "github.com/distribution/distribution/v3/context"
|
|
|
|
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
|
|
|
|
"github.com/distribution/distribution/v3/registry/storage/driver/base"
|
|
|
|
"github.com/distribution/distribution/v3/registry/storage/driver/factory"
|
2014-10-24 23:37:25 +00:00
|
|
|
)
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
const driverName = "s3aws"
|
2014-10-29 01:15:40 +00:00
|
|
|
|
2014-10-29 19:14:19 +00:00
|
|
|
// minChunkSize defines the minimum multipart upload chunk size
|
|
|
|
// S3 API requires multipart upload chunks to be at least 5MB
|
2015-01-24 00:46:43 +00:00
|
|
|
const minChunkSize = 5 << 20
|
|
|
|
|
2016-08-16 00:12:24 +00:00
|
|
|
// maxChunkSize defines the maximum multipart upload chunk size allowed by S3.
|
2023-09-22 13:53:47 +00:00
|
|
|
// S3 API requires max upload chunk to be 5GB.
|
2016-08-16 00:12:24 +00:00
|
|
|
const maxChunkSize = 5 << 30
|
|
|
|
|
2015-01-24 00:46:43 +00:00
|
|
|
const defaultChunkSize = 2 * minChunkSize
|
2014-10-24 23:37:25 +00:00
|
|
|
|
2016-08-16 00:12:24 +00:00
|
|
|
const (
|
|
|
|
// defaultMultipartCopyChunkSize defines the default chunk size for all
|
|
|
|
// but the last Upload Part - Copy operation of a multipart copy.
|
|
|
|
// Empirically, 32 MB is optimal.
|
|
|
|
defaultMultipartCopyChunkSize = 32 << 20
|
|
|
|
|
|
|
|
// defaultMultipartCopyMaxConcurrency defines the default maximum number
|
|
|
|
// of concurrent Upload Part - Copy operations for a multipart copy.
|
|
|
|
defaultMultipartCopyMaxConcurrency = 100
|
|
|
|
|
|
|
|
// defaultMultipartCopyThresholdSize defines the default object size
|
|
|
|
// above which multipart copy will be used. (PUT Object - Copy is used
|
|
|
|
// for objects at or below this size.) Empirically, 32 MB is optimal.
|
|
|
|
defaultMultipartCopyThresholdSize = 32 << 20
|
|
|
|
)
|
|
|
|
|
2014-12-19 17:16:51 +00:00
|
|
|
// listMax is the largest amount of objects you can request from S3 in a list call
|
|
|
|
const listMax = 1000
|
2014-10-24 23:37:25 +00:00
|
|
|
|
2016-09-01 20:52:40 +00:00
|
|
|
// noStorageClass defines the value to be used if storage class is not supported by the S3 endpoint
|
|
|
|
const noStorageClass = "NONE"
|
|
|
|
|
2022-04-04 07:55:51 +00:00
|
|
|
// s3StorageClasses lists all compatible (instant retrieval) S3 storage classes
|
|
|
|
var s3StorageClasses = []string{
|
|
|
|
noStorageClass,
|
|
|
|
s3.StorageClassStandard,
|
|
|
|
s3.StorageClassReducedRedundancy,
|
|
|
|
s3.StorageClassStandardIa,
|
|
|
|
s3.StorageClassOnezoneIa,
|
|
|
|
s3.StorageClassIntelligentTiering,
|
|
|
|
s3.StorageClassOutposts,
|
|
|
|
s3.StorageClassGlacierIr,
|
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
// validRegions maps known s3 region identifiers to region descriptors
|
|
|
|
var validRegions = map[string]struct{}{}
|
|
|
|
|
2016-10-06 00:47:12 +00:00
|
|
|
// validObjectACLs contains known s3 object Acls
|
|
|
|
var validObjectACLs = map[string]struct{}{}
|
2016-07-27 19:26:57 +00:00
|
|
|
|
2022-11-02 21:05:45 +00:00
|
|
|
// DriverParameters A struct that encapsulates all of the driver parameters after all values have been set
|
2015-01-07 10:18:42 +00:00
|
|
|
type DriverParameters struct {
|
2016-08-16 00:12:24 +00:00
|
|
|
AccessKey string
|
|
|
|
SecretKey string
|
|
|
|
Bucket string
|
|
|
|
Region string
|
|
|
|
RegionEndpoint string
|
2022-04-19 14:49:19 +00:00
|
|
|
ForcePathStyle bool
|
2016-08-16 00:12:24 +00:00
|
|
|
Encrypt bool
|
|
|
|
KeyID string
|
|
|
|
Secure bool
|
2017-03-03 19:53:55 +00:00
|
|
|
SkipVerify bool
|
2016-09-01 20:52:40 +00:00
|
|
|
V4Auth bool
|
2016-08-16 00:12:24 +00:00
|
|
|
ChunkSize int64
|
|
|
|
MultipartCopyChunkSize int64
|
|
|
|
MultipartCopyMaxConcurrency int64
|
|
|
|
MultipartCopyThresholdSize int64
|
2021-12-31 06:13:16 +00:00
|
|
|
MultipartCombineSmallPart bool
|
2016-08-16 00:12:24 +00:00
|
|
|
RootDirectory string
|
|
|
|
StorageClass string
|
|
|
|
UserAgent string
|
2016-10-06 00:47:12 +00:00
|
|
|
ObjectACL string
|
2017-03-27 20:04:00 +00:00
|
|
|
SessionToken string
|
2019-12-11 20:41:40 +00:00
|
|
|
UseDualStack bool
|
2021-02-21 14:44:19 +00:00
|
|
|
Accelerate bool
|
2023-09-27 14:57:40 +00:00
|
|
|
LogLevel aws.LogLevelType
|
2015-01-07 10:18:42 +00:00
|
|
|
}
|
|
|
|
|
2014-10-29 01:15:40 +00:00
|
|
|
func init() {
|
2018-08-20 15:22:09 +00:00
|
|
|
partitions := endpoints.DefaultPartitions()
|
|
|
|
for _, p := range partitions {
|
|
|
|
for region := range p.Regions() {
|
|
|
|
validRegions[region] = struct{}{}
|
|
|
|
}
|
2016-01-22 02:17:53 +00:00
|
|
|
}
|
|
|
|
|
2016-10-06 00:47:12 +00:00
|
|
|
for _, objectACL := range []string{
|
2016-07-27 19:26:57 +00:00
|
|
|
s3.ObjectCannedACLPrivate,
|
|
|
|
s3.ObjectCannedACLPublicRead,
|
|
|
|
s3.ObjectCannedACLPublicReadWrite,
|
|
|
|
s3.ObjectCannedACLAuthenticatedRead,
|
|
|
|
s3.ObjectCannedACLAwsExecRead,
|
|
|
|
s3.ObjectCannedACLBucketOwnerRead,
|
|
|
|
s3.ObjectCannedACLBucketOwnerFullControl,
|
|
|
|
} {
|
2016-10-06 00:47:12 +00:00
|
|
|
validObjectACLs[objectACL] = struct{}{}
|
2016-07-27 19:26:57 +00:00
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
// Register this as the default s3 driver in addition to s3aws
|
|
|
|
factory.Register("s3", &s3DriverFactory{})
|
2014-11-17 23:44:07 +00:00
|
|
|
factory.Register(driverName, &s3DriverFactory{})
|
2014-10-29 01:15:40 +00:00
|
|
|
}
|
|
|
|
|
2014-10-29 19:14:19 +00:00
|
|
|
// s3DriverFactory implements the factory.StorageDriverFactory interface
|
2014-10-29 01:15:40 +00:00
|
|
|
type s3DriverFactory struct{}
|
|
|
|
|
2023-10-27 21:33:55 +00:00
|
|
|
func (factory *s3DriverFactory) Create(ctx context.Context, parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
|
|
|
|
return FromParameters(ctx, parameters)
|
2014-10-29 01:15:40 +00:00
|
|
|
}
|
|
|
|
|
2023-08-21 11:54:13 +00:00
|
|
|
var _ storagedriver.StorageDriver = &driver{}
|
|
|
|
|
2015-02-04 00:54:52 +00:00
|
|
|
type driver struct {
|
2016-08-16 00:12:24 +00:00
|
|
|
S3 *s3.S3
|
|
|
|
Bucket string
|
|
|
|
ChunkSize int64
|
|
|
|
Encrypt bool
|
|
|
|
KeyID string
|
|
|
|
MultipartCopyChunkSize int64
|
|
|
|
MultipartCopyMaxConcurrency int64
|
|
|
|
MultipartCopyThresholdSize int64
|
2021-12-31 06:13:16 +00:00
|
|
|
MultipartCombineSmallPart bool
|
2016-08-16 00:12:24 +00:00
|
|
|
RootDirectory string
|
|
|
|
StorageClass string
|
2016-10-06 00:47:12 +00:00
|
|
|
ObjectACL string
|
2023-09-22 13:53:47 +00:00
|
|
|
pool *sync.Pool
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2015-02-04 00:54:52 +00:00
|
|
|
type baseEmbed struct {
|
|
|
|
base.Base
|
|
|
|
}
|
|
|
|
|
|
|
|
// Driver is a storagedriver.StorageDriver implementation backed by Amazon S3
|
|
|
|
// Objects are stored at absolute keys in the provided bucket.
|
|
|
|
type Driver struct {
|
|
|
|
baseEmbed
|
|
|
|
}
|
|
|
|
|
2014-11-17 23:44:07 +00:00
|
|
|
// FromParameters constructs a new Driver with a given parameters map
|
2014-10-29 01:15:40 +00:00
|
|
|
// Required parameters:
|
|
|
|
// - accesskey
|
|
|
|
// - secretkey
|
|
|
|
// - region
|
|
|
|
// - bucket
|
|
|
|
// - encrypt
|
2023-10-27 21:33:55 +00:00
|
|
|
func FromParameters(ctx context.Context, parameters map[string]interface{}) (*Driver, error) {
|
2014-12-22 22:24:45 +00:00
|
|
|
// Providing no values for these is valid in case the user is authenticating
|
|
|
|
// with an IAM on an ec2 instance (in which case the instance credentials will
|
|
|
|
// be summoned when GetAuth is called)
|
2016-03-07 19:50:46 +00:00
|
|
|
accessKey := parameters["accesskey"]
|
|
|
|
if accessKey == nil {
|
2015-01-23 23:50:55 +00:00
|
|
|
accessKey = ""
|
|
|
|
}
|
2016-03-07 19:50:46 +00:00
|
|
|
secretKey := parameters["secretkey"]
|
|
|
|
if secretKey == nil {
|
2015-01-23 23:50:55 +00:00
|
|
|
secretKey = ""
|
|
|
|
}
|
2014-10-29 01:15:40 +00:00
|
|
|
|
2016-04-06 11:34:14 +00:00
|
|
|
regionEndpoint := parameters["regionendpoint"]
|
|
|
|
if regionEndpoint == nil {
|
|
|
|
regionEndpoint = ""
|
|
|
|
}
|
|
|
|
|
2022-04-19 14:49:19 +00:00
|
|
|
forcePathStyleBool := true
|
|
|
|
forcePathStyle := parameters["forcepathstyle"]
|
|
|
|
switch forcePathStyle := forcePathStyle.(type) {
|
|
|
|
case string:
|
|
|
|
b, err := strconv.ParseBool(forcePathStyle)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("the forcePathStyle parameter should be a boolean")
|
|
|
|
}
|
|
|
|
forcePathStyleBool = b
|
|
|
|
case bool:
|
|
|
|
forcePathStyleBool = forcePathStyle
|
|
|
|
case nil:
|
|
|
|
// do nothing
|
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf("the forcePathStyle parameter should be a boolean")
|
|
|
|
}
|
|
|
|
|
2018-08-06 21:34:15 +00:00
|
|
|
regionName := parameters["region"]
|
2016-03-07 19:50:46 +00:00
|
|
|
if regionName == nil || fmt.Sprint(regionName) == "" {
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("no region parameter provided")
|
2014-10-29 01:15:40 +00:00
|
|
|
}
|
2016-01-22 02:17:53 +00:00
|
|
|
region := fmt.Sprint(regionName)
|
2016-04-06 11:34:14 +00:00
|
|
|
// Don't check the region value if a custom endpoint is provided.
|
|
|
|
if regionEndpoint == "" {
|
2018-08-06 21:34:15 +00:00
|
|
|
if _, ok := validRegions[region]; !ok {
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("invalid region provided: %v", region)
|
2016-04-06 11:34:14 +00:00
|
|
|
}
|
2014-10-29 01:15:40 +00:00
|
|
|
}
|
|
|
|
|
2016-03-07 19:50:46 +00:00
|
|
|
bucket := parameters["bucket"]
|
|
|
|
if bucket == nil || fmt.Sprint(bucket) == "" {
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("no bucket parameter provided")
|
2014-10-29 01:15:40 +00:00
|
|
|
}
|
|
|
|
|
2015-01-07 09:51:29 +00:00
|
|
|
encryptBool := false
|
2016-03-07 19:50:46 +00:00
|
|
|
encrypt := parameters["encrypt"]
|
|
|
|
switch encrypt := encrypt.(type) {
|
|
|
|
case string:
|
|
|
|
b, err := strconv.ParseBool(encrypt)
|
|
|
|
if err != nil {
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("the encrypt parameter should be a boolean")
|
2014-12-30 17:31:12 +00:00
|
|
|
}
|
2016-03-07 19:50:46 +00:00
|
|
|
encryptBool = b
|
|
|
|
case bool:
|
|
|
|
encryptBool = encrypt
|
|
|
|
case nil:
|
|
|
|
// do nothing
|
|
|
|
default:
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("the encrypt parameter should be a boolean")
|
2014-10-29 01:15:40 +00:00
|
|
|
}
|
2014-12-19 17:16:51 +00:00
|
|
|
|
2015-01-07 09:45:31 +00:00
|
|
|
secureBool := true
|
2016-03-07 19:50:46 +00:00
|
|
|
secure := parameters["secure"]
|
|
|
|
switch secure := secure.(type) {
|
|
|
|
case string:
|
|
|
|
b, err := strconv.ParseBool(secure)
|
|
|
|
if err != nil {
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("the secure parameter should be a boolean")
|
2014-12-29 20:29:54 +00:00
|
|
|
}
|
2016-03-07 19:50:46 +00:00
|
|
|
secureBool = b
|
|
|
|
case bool:
|
|
|
|
secureBool = secure
|
|
|
|
case nil:
|
|
|
|
// do nothing
|
|
|
|
default:
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("the secure parameter should be a boolean")
|
2015-01-07 09:45:31 +00:00
|
|
|
}
|
|
|
|
|
2017-03-03 19:53:55 +00:00
|
|
|
skipVerifyBool := false
|
|
|
|
skipVerify := parameters["skipverify"]
|
|
|
|
switch skipVerify := skipVerify.(type) {
|
|
|
|
case string:
|
|
|
|
b, err := strconv.ParseBool(skipVerify)
|
|
|
|
if err != nil {
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("the skipVerify parameter should be a boolean")
|
2017-03-03 19:53:55 +00:00
|
|
|
}
|
|
|
|
skipVerifyBool = b
|
|
|
|
case bool:
|
|
|
|
skipVerifyBool = skipVerify
|
|
|
|
case nil:
|
|
|
|
// do nothing
|
|
|
|
default:
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("the skipVerify parameter should be a boolean")
|
2017-03-03 19:53:55 +00:00
|
|
|
}
|
|
|
|
|
2016-09-01 20:52:40 +00:00
|
|
|
v4Bool := true
|
|
|
|
v4auth := parameters["v4auth"]
|
|
|
|
switch v4auth := v4auth.(type) {
|
|
|
|
case string:
|
|
|
|
b, err := strconv.ParseBool(v4auth)
|
|
|
|
if err != nil {
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("the v4auth parameter should be a boolean")
|
2016-09-01 20:52:40 +00:00
|
|
|
}
|
|
|
|
v4Bool = b
|
|
|
|
case bool:
|
|
|
|
v4Bool = v4auth
|
|
|
|
case nil:
|
|
|
|
// do nothing
|
|
|
|
default:
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("the v4auth parameter should be a boolean")
|
2016-09-01 20:52:40 +00:00
|
|
|
}
|
|
|
|
|
2016-03-10 00:52:59 +00:00
|
|
|
keyID := parameters["keyid"]
|
|
|
|
if keyID == nil {
|
|
|
|
keyID = ""
|
|
|
|
}
|
|
|
|
|
2016-08-16 00:12:24 +00:00
|
|
|
chunkSize, err := getParameterAsInt64(parameters, "chunksize", defaultChunkSize, minChunkSize, maxChunkSize)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2016-03-07 19:50:46 +00:00
|
|
|
}
|
2015-04-22 21:31:34 +00:00
|
|
|
|
2016-08-16 00:12:24 +00:00
|
|
|
multipartCopyChunkSize, err := getParameterAsInt64(parameters, "multipartcopychunksize", defaultMultipartCopyChunkSize, minChunkSize, maxChunkSize)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
multipartCopyMaxConcurrency, err := getParameterAsInt64(parameters, "multipartcopymaxconcurrency", defaultMultipartCopyMaxConcurrency, 1, math.MaxInt64)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
multipartCopyThresholdSize, err := getParameterAsInt64(parameters, "multipartcopythresholdsize", defaultMultipartCopyThresholdSize, 0, maxChunkSize)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2015-01-24 00:46:43 +00:00
|
|
|
}
|
|
|
|
|
2016-03-07 19:50:46 +00:00
|
|
|
rootDirectory := parameters["rootdirectory"]
|
|
|
|
if rootDirectory == nil {
|
2014-12-30 17:31:12 +00:00
|
|
|
rootDirectory = ""
|
2014-12-19 17:16:51 +00:00
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
storageClass := s3.StorageClassStandard
|
2016-03-07 19:50:46 +00:00
|
|
|
storageClassParam := parameters["storageclass"]
|
|
|
|
if storageClassParam != nil {
|
2016-01-28 23:48:49 +00:00
|
|
|
storageClassString, ok := storageClassParam.(string)
|
|
|
|
if !ok {
|
2022-04-01 09:51:36 +00:00
|
|
|
return nil, fmt.Errorf(
|
|
|
|
"the storageclass parameter must be one of %v, %v invalid",
|
2022-04-04 07:55:51 +00:00
|
|
|
s3StorageClasses,
|
2022-04-01 09:51:36 +00:00
|
|
|
storageClassParam,
|
|
|
|
)
|
2016-01-28 23:48:49 +00:00
|
|
|
}
|
|
|
|
// All valid storage class parameters are UPPERCASE, so be a bit more flexible here
|
2016-01-22 02:17:53 +00:00
|
|
|
storageClassString = strings.ToUpper(storageClassString)
|
2016-09-01 20:52:40 +00:00
|
|
|
if storageClassString != noStorageClass &&
|
|
|
|
storageClassString != s3.StorageClassStandard &&
|
2022-04-01 09:51:36 +00:00
|
|
|
storageClassString != s3.StorageClassReducedRedundancy &&
|
|
|
|
storageClassString != s3.StorageClassStandardIa &&
|
|
|
|
storageClassString != s3.StorageClassOnezoneIa &&
|
|
|
|
storageClassString != s3.StorageClassIntelligentTiering &&
|
|
|
|
storageClassString != s3.StorageClassOutposts &&
|
|
|
|
storageClassString != s3.StorageClassGlacierIr {
|
|
|
|
return nil, fmt.Errorf(
|
|
|
|
"the storageclass parameter must be one of %v, %v invalid",
|
2022-04-04 07:55:51 +00:00
|
|
|
s3StorageClasses,
|
2022-04-01 09:51:36 +00:00
|
|
|
storageClassParam,
|
|
|
|
)
|
2016-01-28 23:48:49 +00:00
|
|
|
}
|
2016-01-22 02:17:53 +00:00
|
|
|
storageClass = storageClassString
|
2016-01-28 23:48:49 +00:00
|
|
|
}
|
|
|
|
|
2016-03-07 19:50:46 +00:00
|
|
|
userAgent := parameters["useragent"]
|
|
|
|
if userAgent == nil {
|
2016-01-21 00:40:58 +00:00
|
|
|
userAgent = ""
|
|
|
|
}
|
|
|
|
|
2016-10-06 00:47:12 +00:00
|
|
|
objectACL := s3.ObjectCannedACLPrivate
|
|
|
|
objectACLParam := parameters["objectacl"]
|
|
|
|
if objectACLParam != nil {
|
|
|
|
objectACLString, ok := objectACLParam.(string)
|
2016-07-27 19:26:57 +00:00
|
|
|
if !ok {
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("invalid value for objectacl parameter: %v", objectACLParam)
|
2016-07-27 19:26:57 +00:00
|
|
|
}
|
|
|
|
|
2016-10-06 00:47:12 +00:00
|
|
|
if _, ok = validObjectACLs[objectACLString]; !ok {
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("invalid value for objectacl parameter: %v", objectACLParam)
|
2016-07-27 19:26:57 +00:00
|
|
|
}
|
2016-10-06 00:47:12 +00:00
|
|
|
objectACL = objectACLString
|
2016-07-27 19:26:57 +00:00
|
|
|
}
|
|
|
|
|
2019-12-11 20:41:40 +00:00
|
|
|
useDualStackBool := false
|
|
|
|
useDualStack := parameters["usedualstack"]
|
|
|
|
switch useDualStack := useDualStack.(type) {
|
|
|
|
case string:
|
|
|
|
b, err := strconv.ParseBool(useDualStack)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("the useDualStack parameter should be a boolean")
|
|
|
|
}
|
|
|
|
useDualStackBool = b
|
|
|
|
case bool:
|
|
|
|
useDualStackBool = useDualStack
|
|
|
|
case nil:
|
|
|
|
// do nothing
|
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf("the useDualStack parameter should be a boolean")
|
|
|
|
}
|
|
|
|
|
2021-12-31 06:13:16 +00:00
|
|
|
mutlipartCombineSmallPart := true
|
|
|
|
combine := parameters["multipartcombinesmallpart"]
|
|
|
|
switch combine := combine.(type) {
|
|
|
|
case string:
|
|
|
|
b, err := strconv.ParseBool(combine)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("the multipartcombinesmallpart parameter should be a boolean")
|
|
|
|
}
|
|
|
|
mutlipartCombineSmallPart = b
|
|
|
|
case bool:
|
|
|
|
mutlipartCombineSmallPart = combine
|
|
|
|
case nil:
|
|
|
|
// do nothing
|
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf("the multipartcombinesmallpart parameter should be a boolean")
|
|
|
|
}
|
|
|
|
|
2017-03-27 20:04:00 +00:00
|
|
|
sessionToken := ""
|
|
|
|
|
2021-02-21 14:44:19 +00:00
|
|
|
accelerateBool := false
|
|
|
|
accelerate := parameters["accelerate"]
|
|
|
|
switch accelerate := accelerate.(type) {
|
2017-01-20 17:12:37 +00:00
|
|
|
case string:
|
2021-02-21 14:44:19 +00:00
|
|
|
b, err := strconv.ParseBool(accelerate)
|
2017-01-20 17:12:37 +00:00
|
|
|
if err != nil {
|
2021-02-21 14:44:19 +00:00
|
|
|
return nil, fmt.Errorf("the accelerate parameter should be a boolean")
|
2017-01-20 17:12:37 +00:00
|
|
|
}
|
2021-02-21 14:44:19 +00:00
|
|
|
accelerateBool = b
|
2017-01-20 17:12:37 +00:00
|
|
|
case bool:
|
2021-02-21 14:44:19 +00:00
|
|
|
accelerateBool = accelerate
|
2017-01-20 17:12:37 +00:00
|
|
|
case nil:
|
|
|
|
// do nothing
|
|
|
|
default:
|
2021-02-21 14:44:19 +00:00
|
|
|
return nil, fmt.Errorf("the accelerate parameter should be a boolean")
|
2017-01-20 17:12:37 +00:00
|
|
|
}
|
|
|
|
|
2015-01-07 10:18:42 +00:00
|
|
|
params := DriverParameters{
|
|
|
|
fmt.Sprint(accessKey),
|
|
|
|
fmt.Sprint(secretKey),
|
|
|
|
fmt.Sprint(bucket),
|
|
|
|
region,
|
2016-03-05 18:46:44 +00:00
|
|
|
fmt.Sprint(regionEndpoint),
|
2022-04-19 14:49:19 +00:00
|
|
|
forcePathStyleBool,
|
2015-01-07 10:18:42 +00:00
|
|
|
encryptBool,
|
2016-03-10 00:52:59 +00:00
|
|
|
fmt.Sprint(keyID),
|
2015-01-07 10:18:42 +00:00
|
|
|
secureBool,
|
2017-03-03 19:53:55 +00:00
|
|
|
skipVerifyBool,
|
2016-09-01 20:52:40 +00:00
|
|
|
v4Bool,
|
2015-01-24 00:46:43 +00:00
|
|
|
chunkSize,
|
2016-08-16 00:12:24 +00:00
|
|
|
multipartCopyChunkSize,
|
|
|
|
multipartCopyMaxConcurrency,
|
|
|
|
multipartCopyThresholdSize,
|
2021-12-31 06:13:16 +00:00
|
|
|
mutlipartCombineSmallPart,
|
2015-01-07 10:18:42 +00:00
|
|
|
fmt.Sprint(rootDirectory),
|
2016-01-28 23:48:49 +00:00
|
|
|
storageClass,
|
2016-01-21 00:40:58 +00:00
|
|
|
fmt.Sprint(userAgent),
|
2016-10-06 00:47:12 +00:00
|
|
|
objectACL,
|
2017-03-27 20:04:00 +00:00
|
|
|
fmt.Sprint(sessionToken),
|
2019-12-11 20:41:40 +00:00
|
|
|
useDualStackBool,
|
2021-02-21 14:44:19 +00:00
|
|
|
accelerateBool,
|
2023-09-27 14:57:40 +00:00
|
|
|
getS3LogLevelFromParam(parameters["loglevel"]),
|
2015-01-07 10:18:42 +00:00
|
|
|
}
|
|
|
|
|
2023-10-27 21:33:55 +00:00
|
|
|
return New(ctx, params)
|
2014-10-29 01:15:40 +00:00
|
|
|
}
|
|
|
|
|
2023-09-27 14:57:40 +00:00
|
|
|
func getS3LogLevelFromParam(param interface{}) aws.LogLevelType {
|
|
|
|
if param == nil {
|
|
|
|
return aws.LogOff
|
|
|
|
}
|
|
|
|
logLevelParam := param.(string)
|
|
|
|
var logLevel aws.LogLevelType
|
|
|
|
switch strings.ToLower(logLevelParam) {
|
|
|
|
case "off":
|
|
|
|
logLevel = aws.LogOff
|
|
|
|
case "debug":
|
|
|
|
logLevel = aws.LogDebug
|
|
|
|
case "debugwithsigning":
|
|
|
|
logLevel = aws.LogDebugWithSigning
|
|
|
|
case "debugwithhttpbody":
|
|
|
|
logLevel = aws.LogDebugWithHTTPBody
|
|
|
|
case "debugwithrequestretries":
|
|
|
|
logLevel = aws.LogDebugWithRequestRetries
|
|
|
|
case "debugwithrequesterrors":
|
|
|
|
logLevel = aws.LogDebugWithRequestErrors
|
|
|
|
case "debugwitheventstreambody":
|
|
|
|
logLevel = aws.LogDebugWithEventStreamBody
|
|
|
|
default:
|
|
|
|
logLevel = aws.LogOff
|
|
|
|
}
|
|
|
|
return logLevel
|
|
|
|
}
|
|
|
|
|
2019-10-09 12:02:21 +00:00
|
|
|
// getParameterAsInt64 converts parameters[name] to an int64 value (using
|
2016-08-16 00:12:24 +00:00
|
|
|
// defaultt if nil), verifies it is no smaller than min, and returns it.
|
|
|
|
func getParameterAsInt64(parameters map[string]interface{}, name string, defaultt int64, min int64, max int64) (int64, error) {
|
|
|
|
rv := defaultt
|
|
|
|
param := parameters[name]
|
|
|
|
switch v := param.(type) {
|
|
|
|
case string:
|
|
|
|
vv, err := strconv.ParseInt(v, 0, 64)
|
|
|
|
if err != nil {
|
|
|
|
return 0, fmt.Errorf("%s parameter must be an integer, %v invalid", name, param)
|
|
|
|
}
|
|
|
|
rv = vv
|
|
|
|
case int64:
|
|
|
|
rv = v
|
|
|
|
case int, uint, int32, uint32, uint64:
|
|
|
|
rv = reflect.ValueOf(v).Convert(reflect.TypeOf(rv)).Int()
|
|
|
|
case nil:
|
|
|
|
// do nothing
|
|
|
|
default:
|
|
|
|
return 0, fmt.Errorf("invalid value for %s: %#v", name, param)
|
|
|
|
}
|
|
|
|
|
|
|
|
if rv < min || rv > max {
|
2019-02-05 00:01:04 +00:00
|
|
|
return 0, fmt.Errorf("the %s %#v parameter should be a number between %d and %d (inclusive)", name, rv, min, max)
|
2016-08-16 00:12:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return rv, nil
|
|
|
|
}
|
|
|
|
|
2014-11-17 23:44:07 +00:00
|
|
|
// New constructs a new Driver with the given AWS credentials, region, encryption flag, and
|
2014-10-29 19:14:19 +00:00
|
|
|
// bucketName
|
2023-10-27 21:33:55 +00:00
|
|
|
func New(ctx context.Context, params DriverParameters) (*Driver, error) {
|
2016-09-01 20:52:40 +00:00
|
|
|
if !params.V4Auth &&
|
|
|
|
(params.RegionEndpoint == "" ||
|
|
|
|
strings.Contains(params.RegionEndpoint, "s3.amazonaws.com")) {
|
2019-02-05 00:01:04 +00:00
|
|
|
return nil, fmt.Errorf("on Amazon S3 this storage driver can only be used with v4 authentication")
|
2016-09-01 20:52:40 +00:00
|
|
|
}
|
|
|
|
|
2023-09-27 14:57:40 +00:00
|
|
|
awsConfig := aws.NewConfig().WithLogLevel(params.LogLevel)
|
2020-09-02 01:08:42 +00:00
|
|
|
|
|
|
|
if params.AccessKey != "" && params.SecretKey != "" {
|
|
|
|
creds := credentials.NewStaticCredentials(
|
|
|
|
params.AccessKey,
|
|
|
|
params.SecretKey,
|
|
|
|
params.SessionToken,
|
|
|
|
)
|
|
|
|
awsConfig.WithCredentials(creds)
|
2019-08-05 08:13:03 +00:00
|
|
|
}
|
2016-10-13 22:07:37 +00:00
|
|
|
|
|
|
|
if params.RegionEndpoint != "" {
|
2016-03-05 18:46:44 +00:00
|
|
|
awsConfig.WithEndpoint(params.RegionEndpoint)
|
2022-04-19 14:49:19 +00:00
|
|
|
awsConfig.WithS3ForcePathStyle(params.ForcePathStyle)
|
2016-03-05 18:46:44 +00:00
|
|
|
}
|
2014-12-29 20:29:54 +00:00
|
|
|
|
2021-02-21 14:44:19 +00:00
|
|
|
awsConfig.WithS3UseAccelerate(params.Accelerate)
|
2016-01-22 02:17:53 +00:00
|
|
|
awsConfig.WithRegion(params.Region)
|
|
|
|
awsConfig.WithDisableSSL(!params.Secure)
|
2019-12-11 20:41:40 +00:00
|
|
|
if params.UseDualStack {
|
|
|
|
awsConfig.UseDualStackEndpoint = endpoints.DualStackEndpointStateEnabled
|
|
|
|
}
|
2016-01-21 00:40:58 +00:00
|
|
|
|
2023-02-14 15:20:54 +00:00
|
|
|
if params.SkipVerify {
|
|
|
|
httpTransport := &http.Transport{
|
|
|
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
2017-03-03 19:53:55 +00:00
|
|
|
}
|
2023-02-14 15:20:54 +00:00
|
|
|
awsConfig.WithHTTPClient(&http.Client{
|
|
|
|
Transport: httpTransport,
|
|
|
|
})
|
2016-01-21 00:40:58 +00:00
|
|
|
}
|
2014-10-24 23:37:25 +00:00
|
|
|
|
2019-08-05 08:13:03 +00:00
|
|
|
sess, err := session.NewSession(awsConfig)
|
2018-08-06 21:34:15 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to create new session with aws config: %v", err)
|
|
|
|
}
|
2023-02-14 15:20:54 +00:00
|
|
|
|
|
|
|
if params.UserAgent != "" {
|
|
|
|
sess.Handlers.Build.PushBack(request.MakeAddToUserAgentFreeFormHandler(params.UserAgent))
|
|
|
|
}
|
|
|
|
|
2018-08-06 21:34:15 +00:00
|
|
|
s3obj := s3.New(sess)
|
2016-01-21 00:40:58 +00:00
|
|
|
|
2016-09-01 20:52:40 +00:00
|
|
|
// enable S3 compatible signature v2 signing instead
|
|
|
|
if !params.V4Auth {
|
|
|
|
setv2Handlers(s3obj)
|
|
|
|
}
|
|
|
|
|
2014-12-21 06:48:42 +00:00
|
|
|
// TODO Currently multipart uploads have no timestamps, so this would be unwise
|
|
|
|
// if you initiated a new s3driver while another one is running on the same bucket.
|
|
|
|
// multis, _, err := bucket.ListMulti("", "")
|
|
|
|
// if err != nil {
|
|
|
|
// return nil, err
|
|
|
|
// }
|
|
|
|
|
|
|
|
// for _, multi := range multis {
|
|
|
|
// err := multi.Abort()
|
|
|
|
// //TODO appropriate to do this error checking?
|
|
|
|
// if err != nil {
|
|
|
|
// return nil, err
|
|
|
|
// }
|
|
|
|
// }
|
2014-12-19 17:16:51 +00:00
|
|
|
|
2015-02-04 00:54:52 +00:00
|
|
|
d := &driver{
|
2018-09-11 18:40:18 +00:00
|
|
|
S3: s3obj,
|
|
|
|
Bucket: params.Bucket,
|
|
|
|
ChunkSize: params.ChunkSize,
|
|
|
|
Encrypt: params.Encrypt,
|
|
|
|
KeyID: params.KeyID,
|
2016-08-16 00:12:24 +00:00
|
|
|
MultipartCopyChunkSize: params.MultipartCopyChunkSize,
|
|
|
|
MultipartCopyMaxConcurrency: params.MultipartCopyMaxConcurrency,
|
|
|
|
MultipartCopyThresholdSize: params.MultipartCopyThresholdSize,
|
2021-12-31 06:13:16 +00:00
|
|
|
MultipartCombineSmallPart: params.MultipartCombineSmallPart,
|
2016-08-16 00:12:24 +00:00
|
|
|
RootDirectory: params.RootDirectory,
|
|
|
|
StorageClass: params.StorageClass,
|
2016-10-06 00:47:12 +00:00
|
|
|
ObjectACL: params.ObjectACL,
|
2023-09-22 13:53:47 +00:00
|
|
|
pool: &sync.Pool{
|
|
|
|
New: func() interface{} {
|
|
|
|
return &buffer{
|
|
|
|
data: make([]byte, 0, params.ChunkSize),
|
|
|
|
}
|
|
|
|
},
|
|
|
|
},
|
2015-02-04 00:54:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return &Driver{
|
|
|
|
baseEmbed: baseEmbed{
|
|
|
|
Base: base.Base{
|
|
|
|
StorageDriver: d,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}, nil
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2014-10-29 19:14:19 +00:00
|
|
|
// Implement the storagedriver.StorageDriver interface
|
|
|
|
|
2015-04-23 00:30:01 +00:00
|
|
|
func (d *driver) Name() string {
|
|
|
|
return driverName
|
|
|
|
}
|
|
|
|
|
2014-11-17 23:44:07 +00:00
|
|
|
// GetContent retrieves the content stored at "path" as a []byte.
|
2015-04-27 22:58:58 +00:00
|
|
|
func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) {
|
2016-02-08 22:29:21 +00:00
|
|
|
reader, err := d.Reader(ctx, path, 0)
|
2014-11-19 01:41:48 +00:00
|
|
|
if err != nil {
|
2016-01-22 02:17:53 +00:00
|
|
|
return nil, err
|
2014-11-19 01:41:48 +00:00
|
|
|
}
|
2022-11-02 21:55:22 +00:00
|
|
|
return io.ReadAll(reader)
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2014-11-17 23:44:07 +00:00
|
|
|
// PutContent stores the []byte content at a location designated by "path".
|
2015-04-27 22:58:58 +00:00
|
|
|
func (d *driver) PutContent(ctx context.Context, path string, contents []byte) error {
|
2023-09-03 20:54:54 +00:00
|
|
|
_, err := d.S3.PutObjectWithContext(ctx, &s3.PutObjectInput{
|
2016-01-22 02:17:53 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Key: aws.String(d.s3Path(path)),
|
|
|
|
ContentType: d.getContentType(),
|
|
|
|
ACL: d.getACL(),
|
|
|
|
ServerSideEncryption: d.getEncryptionMode(),
|
2016-03-10 00:52:59 +00:00
|
|
|
SSEKMSKeyId: d.getSSEKMSKeyID(),
|
2016-01-22 02:17:53 +00:00
|
|
|
StorageClass: d.getStorageClass(),
|
|
|
|
Body: bytes.NewReader(contents),
|
|
|
|
})
|
|
|
|
return parseError(path, err)
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2016-02-08 22:29:21 +00:00
|
|
|
// Reader retrieves an io.ReadCloser for the content stored at "path" with a
|
2014-11-17 23:44:07 +00:00
|
|
|
// given byte offset.
|
2016-02-08 22:29:21 +00:00
|
|
|
func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
|
2023-09-03 20:54:54 +00:00
|
|
|
resp, err := d.S3.GetObjectWithContext(ctx, &s3.GetObjectInput{
|
2016-01-22 02:17:53 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Key: aws.String(d.s3Path(path)),
|
|
|
|
Range: aws.String("bytes=" + strconv.FormatInt(offset, 10) + "-"),
|
|
|
|
})
|
2014-11-19 01:41:48 +00:00
|
|
|
if err != nil {
|
2016-01-22 02:17:53 +00:00
|
|
|
if s3Err, ok := err.(awserr.Error); ok && s3Err.Code() == "InvalidRange" {
|
2022-11-02 21:55:22 +00:00
|
|
|
return io.NopCloser(bytes.NewReader(nil)), nil
|
2014-12-19 17:16:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil, parseError(path, err)
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
2014-11-19 01:41:48 +00:00
|
|
|
return resp.Body, nil
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2016-02-08 22:29:21 +00:00
|
|
|
// Writer returns a FileWriter which will store the content written to it
|
|
|
|
// at the location designated by "path" after the call to Commit.
|
2019-01-11 16:12:23 +00:00
|
|
|
func (d *driver) Writer(ctx context.Context, path string, appendParam bool) (storagedriver.FileWriter, error) {
|
2016-02-08 22:29:21 +00:00
|
|
|
key := d.s3Path(path)
|
2019-01-11 16:12:23 +00:00
|
|
|
if !appendParam {
|
2016-02-08 22:29:21 +00:00
|
|
|
// TODO (brianbland): cancel other uploads at this path
|
2023-09-03 20:54:54 +00:00
|
|
|
resp, err := d.S3.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Key: aws.String(key),
|
|
|
|
ContentType: d.getContentType(),
|
|
|
|
ACL: d.getACL(),
|
|
|
|
ServerSideEncryption: d.getEncryptionMode(),
|
2016-03-10 00:52:59 +00:00
|
|
|
SSEKMSKeyId: d.getSSEKMSKeyID(),
|
2016-02-08 22:29:21 +00:00
|
|
|
StorageClass: d.getStorageClass(),
|
|
|
|
})
|
2014-12-19 17:16:51 +00:00
|
|
|
if err != nil {
|
2016-02-08 22:29:21 +00:00
|
|
|
return nil, err
|
2014-12-19 17:20:07 +00:00
|
|
|
}
|
2023-10-18 09:34:10 +00:00
|
|
|
return d.newWriter(ctx, key, *resp.UploadId, nil), nil
|
2014-12-19 17:20:07 +00:00
|
|
|
}
|
2021-10-26 15:41:11 +00:00
|
|
|
|
|
|
|
listMultipartUploadsInput := &s3.ListMultipartUploadsInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Prefix: aws.String(key),
|
2014-12-19 17:20:07 +00:00
|
|
|
}
|
2021-10-26 15:41:11 +00:00
|
|
|
for {
|
2023-09-03 20:54:54 +00:00
|
|
|
resp, err := d.S3.ListMultipartUploadsWithContext(ctx, listMultipartUploadsInput)
|
2014-12-19 17:20:07 +00:00
|
|
|
if err != nil {
|
2016-02-08 22:29:21 +00:00
|
|
|
return nil, parseError(path, err)
|
2014-12-19 17:16:51 +00:00
|
|
|
}
|
2021-10-26 15:41:11 +00:00
|
|
|
|
|
|
|
// resp.Uploads can only be empty on the first call
|
|
|
|
// if there were no more results to return after the first call, resp.IsTruncated would have been false
|
|
|
|
// and the loop would be exited without recalling ListMultipartUploads
|
|
|
|
if len(resp.Uploads) == 0 {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
var allParts []*s3.Part
|
|
|
|
for _, multi := range resp.Uploads {
|
|
|
|
if key != *multi.Key {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2023-09-03 20:54:54 +00:00
|
|
|
partsList, err := d.S3.ListPartsWithContext(ctx, &s3.ListPartsInput{
|
2021-10-26 15:41:11 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Key: aws.String(key),
|
|
|
|
UploadId: multi.UploadId,
|
2019-01-11 16:12:23 +00:00
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return nil, parseError(path, err)
|
|
|
|
}
|
2021-10-26 15:41:11 +00:00
|
|
|
allParts = append(allParts, partsList.Parts...)
|
2023-02-22 04:57:50 +00:00
|
|
|
for *partsList.IsTruncated {
|
2023-09-03 20:54:54 +00:00
|
|
|
partsList, err = d.S3.ListPartsWithContext(ctx, &s3.ListPartsInput{
|
2021-10-26 15:41:11 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Key: aws.String(key),
|
|
|
|
UploadId: multi.UploadId,
|
|
|
|
PartNumberMarker: partsList.NextPartNumberMarker,
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return nil, parseError(path, err)
|
|
|
|
}
|
|
|
|
allParts = append(allParts, partsList.Parts...)
|
|
|
|
}
|
2023-10-18 09:34:10 +00:00
|
|
|
return d.newWriter(ctx, key, *multi.UploadId, allParts), nil
|
2021-10-26 15:41:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// resp.NextUploadIdMarker must have at least one element or we would have returned not found
|
|
|
|
listMultipartUploadsInput.UploadIdMarker = resp.NextUploadIdMarker
|
|
|
|
|
|
|
|
// from the s3 api docs, IsTruncated "specifies whether (true) or not (false) all of the results were returned"
|
|
|
|
// if everything has been returned, break
|
|
|
|
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
|
|
|
break
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
}
|
2016-02-08 22:29:21 +00:00
|
|
|
return nil, storagedriver.PathNotFoundError{Path: path}
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2014-12-19 17:16:51 +00:00
|
|
|
// Stat retrieves the FileInfo for the given path, including the current size
|
|
|
|
// in bytes and the creation time.
|
2015-04-27 22:58:58 +00:00
|
|
|
func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
|
2023-09-03 20:54:54 +00:00
|
|
|
resp, err := d.S3.ListObjectsV2WithContext(ctx, &s3.ListObjectsV2Input{
|
2016-01-22 02:17:53 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Prefix: aws.String(d.s3Path(path)),
|
|
|
|
MaxKeys: aws.Int64(1),
|
|
|
|
})
|
2014-10-24 23:37:25 +00:00
|
|
|
if err != nil {
|
2014-12-19 17:16:51 +00:00
|
|
|
return nil, err
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2014-12-19 17:16:51 +00:00
|
|
|
fi := storagedriver.FileInfoFields{
|
|
|
|
Path: path,
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
if len(resp.Contents) == 1 {
|
|
|
|
if *resp.Contents[0].Key != d.s3Path(path) {
|
2014-12-19 17:16:51 +00:00
|
|
|
fi.IsDir = true
|
|
|
|
} else {
|
|
|
|
fi.IsDir = false
|
2016-01-22 02:17:53 +00:00
|
|
|
fi.Size = *resp.Contents[0].Size
|
|
|
|
fi.ModTime = *resp.Contents[0].LastModified
|
2014-12-19 17:16:51 +00:00
|
|
|
}
|
2016-01-22 02:17:53 +00:00
|
|
|
} else if len(resp.CommonPrefixes) == 1 {
|
2014-12-19 17:16:51 +00:00
|
|
|
fi.IsDir = true
|
|
|
|
} else {
|
|
|
|
return nil, storagedriver.PathNotFoundError{Path: path}
|
|
|
|
}
|
|
|
|
|
|
|
|
return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2014-12-19 17:16:51 +00:00
|
|
|
// List returns a list of the objects that are direct descendants of the given path.
|
2015-12-08 19:02:40 +00:00
|
|
|
func (d *driver) List(ctx context.Context, opath string) ([]string, error) {
|
|
|
|
path := opath
|
2014-12-19 17:16:51 +00:00
|
|
|
if path != "/" && path[len(path)-1] != '/' {
|
2014-11-07 20:58:48 +00:00
|
|
|
path = path + "/"
|
2014-11-04 00:20:38 +00:00
|
|
|
}
|
2015-02-20 00:31:34 +00:00
|
|
|
|
|
|
|
// This is to cover for the cases when the rootDirectory of the driver is either "" or "/".
|
|
|
|
// In those cases, there is no root prefix to replace and we must actually add a "/" to all
|
|
|
|
// results in order to keep them as valid paths as recognized by storagedriver.PathRegexp
|
|
|
|
prefix := ""
|
|
|
|
if d.s3Path("") == "" {
|
|
|
|
prefix = "/"
|
|
|
|
}
|
|
|
|
|
2023-09-03 20:54:54 +00:00
|
|
|
resp, err := d.S3.ListObjectsV2WithContext(ctx, &s3.ListObjectsV2Input{
|
2016-01-22 02:17:53 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Prefix: aws.String(d.s3Path(path)),
|
|
|
|
Delimiter: aws.String("/"),
|
|
|
|
MaxKeys: aws.Int64(listMax),
|
|
|
|
})
|
2014-10-24 23:37:25 +00:00
|
|
|
if err != nil {
|
2015-12-08 19:02:40 +00:00
|
|
|
return nil, parseError(opath, err)
|
2015-11-24 22:23:12 +00:00
|
|
|
}
|
|
|
|
|
2014-10-24 23:37:25 +00:00
|
|
|
files := []string{}
|
|
|
|
directories := []string{}
|
|
|
|
|
2014-11-04 00:20:38 +00:00
|
|
|
for {
|
2016-01-22 02:17:53 +00:00
|
|
|
for _, key := range resp.Contents {
|
|
|
|
files = append(files, strings.Replace(*key.Key, d.s3Path(""), prefix, 1))
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
for _, commonPrefix := range resp.CommonPrefixes {
|
|
|
|
commonPrefix := *commonPrefix.Prefix
|
2015-02-20 00:31:34 +00:00
|
|
|
directories = append(directories, strings.Replace(commonPrefix[0:len(commonPrefix)-1], d.s3Path(""), prefix, 1))
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
if *resp.IsTruncated {
|
2023-09-03 20:54:54 +00:00
|
|
|
resp, err = d.S3.ListObjectsV2WithContext(ctx, &s3.ListObjectsV2Input{
|
2022-04-09 11:16:46 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Prefix: aws.String(d.s3Path(path)),
|
|
|
|
Delimiter: aws.String("/"),
|
|
|
|
MaxKeys: aws.Int64(listMax),
|
|
|
|
ContinuationToken: resp.NextContinuationToken,
|
2016-01-22 02:17:53 +00:00
|
|
|
})
|
2014-11-04 00:20:38 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2014-10-24 23:37:25 +00:00
|
|
|
} else {
|
2014-11-04 00:20:38 +00:00
|
|
|
break
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-12-08 19:02:40 +00:00
|
|
|
if opath != "/" {
|
|
|
|
if len(files) == 0 && len(directories) == 0 {
|
|
|
|
// Treat empty response as missing directory, since we don't actually
|
|
|
|
// have directories in s3.
|
|
|
|
return nil, storagedriver.PathNotFoundError{Path: opath}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-10-24 23:37:25 +00:00
|
|
|
return append(files, directories...), nil
|
|
|
|
}
|
|
|
|
|
2014-11-17 23:44:07 +00:00
|
|
|
// Move moves an object stored at sourcePath to destPath, removing the original
|
|
|
|
// object.
|
2015-04-27 22:58:58 +00:00
|
|
|
func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error {
|
2014-10-24 23:37:25 +00:00
|
|
|
/* This is terrible, but aws doesn't have an actual move. */
|
2016-08-16 00:12:24 +00:00
|
|
|
if err := d.copy(ctx, sourcePath, destPath); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return d.Delete(ctx, sourcePath)
|
|
|
|
}
|
|
|
|
|
|
|
|
// copy copies an object stored at sourcePath to destPath.
|
|
|
|
func (d *driver) copy(ctx context.Context, sourcePath string, destPath string) error {
|
|
|
|
// S3 can copy objects up to 5 GB in size with a single PUT Object - Copy
|
|
|
|
// operation. For larger objects, the multipart upload API must be used.
|
|
|
|
//
|
|
|
|
// Empirically, multipart copy is fastest with 32 MB parts and is faster
|
|
|
|
// than PUT Object - Copy for objects larger than 32 MB.
|
|
|
|
|
|
|
|
fileInfo, err := d.Stat(ctx, sourcePath)
|
|
|
|
if err != nil {
|
|
|
|
return parseError(sourcePath, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if fileInfo.Size() <= d.MultipartCopyThresholdSize {
|
2023-09-03 20:54:54 +00:00
|
|
|
_, err := d.S3.CopyObjectWithContext(ctx, &s3.CopyObjectInput{
|
2016-08-16 00:12:24 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Key: aws.String(d.s3Path(destPath)),
|
|
|
|
ContentType: d.getContentType(),
|
|
|
|
ACL: d.getACL(),
|
|
|
|
ServerSideEncryption: d.getEncryptionMode(),
|
|
|
|
SSEKMSKeyId: d.getSSEKMSKeyID(),
|
|
|
|
StorageClass: d.getStorageClass(),
|
|
|
|
CopySource: aws.String(d.Bucket + "/" + d.s3Path(sourcePath)),
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return parseError(sourcePath, err)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-09-03 20:54:54 +00:00
|
|
|
createResp, err := d.S3.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{
|
2016-01-22 02:17:53 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Key: aws.String(d.s3Path(destPath)),
|
|
|
|
ContentType: d.getContentType(),
|
|
|
|
ACL: d.getACL(),
|
2016-03-10 00:52:59 +00:00
|
|
|
SSEKMSKeyId: d.getSSEKMSKeyID(),
|
2016-08-16 00:12:24 +00:00
|
|
|
ServerSideEncryption: d.getEncryptionMode(),
|
2016-01-22 02:17:53 +00:00
|
|
|
StorageClass: d.getStorageClass(),
|
|
|
|
})
|
2014-10-24 23:37:25 +00:00
|
|
|
if err != nil {
|
2016-08-16 00:12:24 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
numParts := (fileInfo.Size() + d.MultipartCopyChunkSize - 1) / d.MultipartCopyChunkSize
|
|
|
|
completedParts := make([]*s3.CompletedPart, numParts)
|
|
|
|
errChan := make(chan error, numParts)
|
|
|
|
limiter := make(chan struct{}, d.MultipartCopyMaxConcurrency)
|
|
|
|
|
|
|
|
for i := range completedParts {
|
|
|
|
i := int64(i)
|
|
|
|
go func() {
|
|
|
|
limiter <- struct{}{}
|
|
|
|
firstByte := i * d.MultipartCopyChunkSize
|
|
|
|
lastByte := firstByte + d.MultipartCopyChunkSize - 1
|
|
|
|
if lastByte >= fileInfo.Size() {
|
|
|
|
lastByte = fileInfo.Size() - 1
|
|
|
|
}
|
2023-09-03 20:54:54 +00:00
|
|
|
uploadResp, err := d.S3.UploadPartCopyWithContext(ctx, &s3.UploadPartCopyInput{
|
2016-08-16 00:12:24 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
CopySource: aws.String(d.Bucket + "/" + d.s3Path(sourcePath)),
|
|
|
|
Key: aws.String(d.s3Path(destPath)),
|
|
|
|
PartNumber: aws.Int64(i + 1),
|
|
|
|
UploadId: createResp.UploadId,
|
|
|
|
CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", firstByte, lastByte)),
|
|
|
|
})
|
|
|
|
if err == nil {
|
|
|
|
completedParts[i] = &s3.CompletedPart{
|
|
|
|
ETag: uploadResp.CopyPartResult.ETag,
|
|
|
|
PartNumber: aws.Int64(i + 1),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
errChan <- err
|
|
|
|
<-limiter
|
|
|
|
}()
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2016-08-16 00:12:24 +00:00
|
|
|
for range completedParts {
|
|
|
|
err := <-errChan
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-03 20:54:54 +00:00
|
|
|
_, err = d.S3.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{
|
2016-08-16 00:12:24 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Key: aws.String(d.s3Path(destPath)),
|
|
|
|
UploadId: createResp.UploadId,
|
|
|
|
MultipartUpload: &s3.CompletedMultipartUpload{Parts: completedParts},
|
|
|
|
})
|
|
|
|
return err
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2014-11-17 23:44:07 +00:00
|
|
|
// Delete recursively deletes all objects stored at "path" and its subpaths.
|
2016-06-28 00:39:25 +00:00
|
|
|
// We must be careful since S3 does not guarantee read after delete consistency
|
2015-04-27 22:58:58 +00:00
|
|
|
func (d *driver) Delete(ctx context.Context, path string) error {
|
2016-06-28 00:39:25 +00:00
|
|
|
s3Objects := make([]*s3.ObjectIdentifier, 0, listMax)
|
2022-04-23 11:37:09 +00:00
|
|
|
s3Path := d.s3Path(path)
|
2022-04-09 11:16:46 +00:00
|
|
|
listObjectsInput := &s3.ListObjectsV2Input{
|
2016-01-22 02:17:53 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
2016-10-05 23:39:38 +00:00
|
|
|
Prefix: aws.String(s3Path),
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
2022-04-23 11:37:09 +00:00
|
|
|
|
2016-06-28 00:39:25 +00:00
|
|
|
for {
|
|
|
|
// list all the objects
|
2023-09-03 20:54:54 +00:00
|
|
|
resp, err := d.S3.ListObjectsV2WithContext(ctx, listObjectsInput)
|
2016-06-28 00:39:25 +00:00
|
|
|
|
|
|
|
// resp.Contents can only be empty on the first call
|
|
|
|
// if there were no more results to return after the first call, resp.IsTruncated would have been false
|
2022-04-23 11:37:09 +00:00
|
|
|
// and the loop would exit without recalling ListObjects
|
2016-06-28 00:39:25 +00:00
|
|
|
if err != nil || len(resp.Contents) == 0 {
|
2022-04-23 11:37:09 +00:00
|
|
|
return storagedriver.PathNotFoundError{Path: path}
|
2016-06-28 00:39:25 +00:00
|
|
|
}
|
2014-10-24 23:37:25 +00:00
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
for _, key := range resp.Contents {
|
2022-04-23 11:37:09 +00:00
|
|
|
// Skip if we encounter a key that is not a subpath (so that deleting "/a" does not delete "/ab").
|
|
|
|
if len(*key.Key) > len(s3Path) && (*key.Key)[len(s3Path)] != '/' {
|
|
|
|
continue
|
|
|
|
}
|
2016-01-22 02:17:53 +00:00
|
|
|
s3Objects = append(s3Objects, &s3.ObjectIdentifier{
|
|
|
|
Key: key.Key,
|
|
|
|
})
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2022-04-23 11:37:09 +00:00
|
|
|
// Delete objects only if the list is not empty, otherwise S3 API returns a cryptic error
|
|
|
|
if len(s3Objects) > 0 {
|
|
|
|
// NOTE: according to AWS docs https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
|
|
|
|
// by default the response returns up to 1,000 key names. The response _might_ contain fewer keys but it will never contain more.
|
|
|
|
// 10000 keys is coincidentally (?) also the max number of keys that can be deleted in a single Delete operation, so we'll just smack
|
|
|
|
// Delete here straight away and reset the object slice when successful.
|
2023-09-03 20:54:54 +00:00
|
|
|
resp, err := d.S3.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
|
2022-04-23 11:37:09 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Delete: &s3.Delete{
|
|
|
|
Objects: s3Objects,
|
|
|
|
Quiet: aws.Bool(false),
|
|
|
|
},
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(resp.Errors) > 0 {
|
|
|
|
// NOTE: AWS SDK s3.Error does not implement error interface which
|
|
|
|
// is pretty intensely sad, so we have to do away with this for now.
|
|
|
|
errs := make([]error, 0, len(resp.Errors))
|
|
|
|
for _, err := range resp.Errors {
|
|
|
|
errs = append(errs, errors.New(err.String()))
|
|
|
|
}
|
|
|
|
return storagedriver.Errors{
|
|
|
|
DriverName: driverName,
|
|
|
|
Errs: errs,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// NOTE: we don't want to reallocate
|
|
|
|
// the slice so we simply "reset" it
|
|
|
|
s3Objects = s3Objects[:0]
|
|
|
|
|
2016-06-28 00:39:25 +00:00
|
|
|
// resp.Contents must have at least one element or we would have returned not found
|
2022-04-09 11:16:46 +00:00
|
|
|
listObjectsInput.StartAfter = resp.Contents[len(resp.Contents)-1].Key
|
2016-06-28 00:39:25 +00:00
|
|
|
|
|
|
|
// from the s3 api docs, IsTruncated "specifies whether (true) or not (false) all of the results were returned"
|
|
|
|
// if everything has been returned, break
|
|
|
|
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-10-24 23:37:25 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2015-01-07 16:31:38 +00:00
|
|
|
// URLFor returns a URL which may be used to retrieve the content stored at the given path.
|
|
|
|
// May return an UnsupportedMethodErr in certain StorageDriver implementations.
|
2015-04-27 22:58:58 +00:00
|
|
|
func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
|
2022-11-02 22:31:23 +00:00
|
|
|
methodString := http.MethodGet
|
2015-01-14 19:31:11 +00:00
|
|
|
method, ok := options["method"]
|
|
|
|
if ok {
|
|
|
|
methodString, ok = method.(string)
|
2022-11-02 22:31:23 +00:00
|
|
|
if !ok || (methodString != http.MethodGet && methodString != http.MethodHead) {
|
2015-11-02 21:23:53 +00:00
|
|
|
return "", storagedriver.ErrUnsupportedMethod{}
|
2015-01-14 19:31:11 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
expiresIn := 20 * time.Minute
|
2015-01-09 01:45:21 +00:00
|
|
|
expires, ok := options["expiry"]
|
2015-01-09 01:10:32 +00:00
|
|
|
if ok {
|
|
|
|
et, ok := expires.(time.Time)
|
|
|
|
if ok {
|
2019-02-05 00:01:04 +00:00
|
|
|
expiresIn = time.Until(et)
|
2015-01-09 01:10:32 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
var req *request.Request
|
|
|
|
|
|
|
|
switch methodString {
|
2022-11-02 22:31:23 +00:00
|
|
|
case http.MethodGet:
|
2016-01-22 02:17:53 +00:00
|
|
|
req, _ = d.S3.GetObjectRequest(&s3.GetObjectInput{
|
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Key: aws.String(d.s3Path(path)),
|
|
|
|
})
|
2022-11-02 22:31:23 +00:00
|
|
|
case http.MethodHead:
|
2016-01-22 02:17:53 +00:00
|
|
|
req, _ = d.S3.HeadObjectRequest(&s3.HeadObjectInput{
|
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Key: aws.String(d.s3Path(path)),
|
|
|
|
})
|
|
|
|
default:
|
|
|
|
panic("unreachable")
|
|
|
|
}
|
|
|
|
|
|
|
|
return req.Presign(expiresIn)
|
2015-01-07 16:31:38 +00:00
|
|
|
}
|
|
|
|
|
2017-11-29 19:17:39 +00:00
|
|
|
// Walk traverses a filesystem defined within driver, starting
|
|
|
|
// from the given path, calling f on each file
|
2022-07-10 02:04:50 +00:00
|
|
|
func (d *driver) Walk(ctx context.Context, from string, f storagedriver.WalkFn, options ...func(*storagedriver.WalkOptions)) error {
|
|
|
|
walkOptions := &storagedriver.WalkOptions{}
|
|
|
|
for _, o := range options {
|
|
|
|
o(walkOptions)
|
2017-11-29 20:26:28 +00:00
|
|
|
}
|
|
|
|
|
2022-07-10 02:04:50 +00:00
|
|
|
var objectCount int64
|
|
|
|
if err := d.doWalk(ctx, &objectCount, from, walkOptions.StartAfterHint, f); err != nil {
|
|
|
|
return err
|
2017-11-29 20:26:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-07-10 02:04:50 +00:00
|
|
|
func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, from string, startAfter string, f storagedriver.WalkFn) error {
|
2021-06-24 18:42:02 +00:00
|
|
|
var (
|
|
|
|
retError error
|
|
|
|
// the most recent directory walked for de-duping
|
|
|
|
prevDir string
|
|
|
|
// the most recent skip directory to avoid walking over undesirable files
|
|
|
|
prevSkipDir string
|
|
|
|
)
|
2022-08-12 15:08:40 +00:00
|
|
|
prevDir = from
|
|
|
|
|
|
|
|
path := from
|
|
|
|
if !strings.HasSuffix(path, "/") {
|
|
|
|
path = path + "/"
|
|
|
|
}
|
|
|
|
|
|
|
|
prefix := ""
|
|
|
|
if d.s3Path("") == "" {
|
|
|
|
prefix = "/"
|
|
|
|
}
|
2017-11-29 20:26:28 +00:00
|
|
|
|
|
|
|
listObjectsInput := &s3.ListObjectsV2Input{
|
2022-07-10 02:04:50 +00:00
|
|
|
Bucket: aws.String(d.Bucket),
|
|
|
|
Prefix: aws.String(d.s3Path(path)),
|
|
|
|
MaxKeys: aws.Int64(listMax),
|
|
|
|
StartAfter: aws.String(d.s3Path(startAfter)),
|
2017-11-29 20:26:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ctx, done := dcontext.WithTrace(parentCtx)
|
2022-07-10 02:04:50 +00:00
|
|
|
defer done("s3aws.ListObjectsV2PagesWithContext(%s)", listObjectsInput)
|
2018-04-25 14:31:07 +00:00
|
|
|
|
2021-06-24 18:42:02 +00:00
|
|
|
// When the "delimiter" argument is omitted, the S3 list API will list all objects in the bucket
|
|
|
|
// recursively, omitting directory paths. Objects are listed in sorted, depth-first order so we
|
|
|
|
// can infer all the directories by comparing each object path to the last one we saw.
|
|
|
|
// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ListingKeysUsingAPIs.html
|
2017-11-29 20:26:28 +00:00
|
|
|
|
2021-06-24 18:42:02 +00:00
|
|
|
// With files returned in sorted depth-first order, directories are inferred in the same order.
|
|
|
|
// ErrSkipDir is handled by explicitly skipping over any files under the skipped directory. This may be sub-optimal
|
|
|
|
// for extreme edge cases but for the general use case in a registry, this is orders of magnitude
|
|
|
|
// faster than a more explicit recursive implementation.
|
|
|
|
listObjectErr := d.S3.ListObjectsV2PagesWithContext(ctx, listObjectsInput, func(objects *s3.ListObjectsV2Output, lastPage bool) bool {
|
|
|
|
walkInfos := make([]storagedriver.FileInfoInternal, 0, len(objects.Contents))
|
2017-11-29 20:26:28 +00:00
|
|
|
|
|
|
|
for _, file := range objects.Contents {
|
2021-06-24 18:42:02 +00:00
|
|
|
filePath := strings.Replace(*file.Key, d.s3Path(""), prefix, 1)
|
|
|
|
|
|
|
|
// get a list of all inferred directories between the previous directory and this file
|
|
|
|
dirs := directoryDiff(prevDir, filePath)
|
|
|
|
if len(dirs) > 0 {
|
|
|
|
for _, dir := range dirs {
|
|
|
|
walkInfos = append(walkInfos, storagedriver.FileInfoInternal{
|
|
|
|
FileInfoFields: storagedriver.FileInfoFields{
|
|
|
|
IsDir: true,
|
|
|
|
Path: dir,
|
|
|
|
},
|
|
|
|
})
|
|
|
|
prevDir = dir
|
|
|
|
}
|
2020-11-30 12:04:14 +00:00
|
|
|
}
|
2021-06-24 18:42:02 +00:00
|
|
|
|
|
|
|
walkInfos = append(walkInfos, storagedriver.FileInfoInternal{
|
2017-11-29 20:26:28 +00:00
|
|
|
FileInfoFields: storagedriver.FileInfoFields{
|
|
|
|
IsDir: false,
|
|
|
|
Size: *file.Size,
|
|
|
|
ModTime: *file.LastModified,
|
2021-06-24 18:42:02 +00:00
|
|
|
Path: filePath,
|
2017-11-29 20:26:28 +00:00
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, walkInfo := range walkInfos {
|
2021-06-24 18:42:02 +00:00
|
|
|
// skip any results under the last skip directory
|
|
|
|
if prevSkipDir != "" && strings.HasPrefix(walkInfo.Path(), prevSkipDir) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2017-11-29 20:26:28 +00:00
|
|
|
err := f(walkInfo)
|
2021-06-24 18:42:02 +00:00
|
|
|
*objectCount++
|
2017-11-29 20:26:28 +00:00
|
|
|
|
2021-06-24 18:42:02 +00:00
|
|
|
if err != nil {
|
|
|
|
if err == storagedriver.ErrSkipDir {
|
2022-07-10 02:04:50 +00:00
|
|
|
prevSkipDir = walkInfo.Path()
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if err == storagedriver.ErrFilledBuffer {
|
2021-06-24 18:42:02 +00:00
|
|
|
return false
|
2017-11-29 20:26:28 +00:00
|
|
|
}
|
|
|
|
retError = err
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
})
|
|
|
|
|
|
|
|
if retError != nil {
|
|
|
|
return retError
|
|
|
|
}
|
|
|
|
|
|
|
|
if listObjectErr != nil {
|
|
|
|
return listObjectErr
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
2017-11-29 19:17:39 +00:00
|
|
|
}
|
|
|
|
|
2021-06-24 18:42:02 +00:00
|
|
|
// directoryDiff finds all directories that are not in common between
|
|
|
|
// the previous and current paths in sorted order.
|
|
|
|
//
|
2022-11-02 21:05:45 +00:00
|
|
|
// # Examples
|
|
|
|
//
|
|
|
|
// directoryDiff("/path/to/folder", "/path/to/folder/folder/file")
|
|
|
|
// // => [ "/path/to/folder/folder" ]
|
|
|
|
//
|
|
|
|
// directoryDiff("/path/to/folder/folder1", "/path/to/folder/folder2/file")
|
|
|
|
// // => [ "/path/to/folder/folder2" ]
|
|
|
|
//
|
|
|
|
// directoryDiff("/path/to/folder/folder1/file", "/path/to/folder/folder2/file")
|
|
|
|
// // => [ "/path/to/folder/folder2" ]
|
|
|
|
//
|
|
|
|
// directoryDiff("/path/to/folder/folder1/file", "/path/to/folder/folder2/folder1/file")
|
|
|
|
// // => [ "/path/to/folder/folder2", "/path/to/folder/folder2/folder1" ]
|
|
|
|
//
|
|
|
|
// directoryDiff("/", "/path/to/folder/folder/file")
|
|
|
|
// // => [ "/path", "/path/to", "/path/to/folder", "/path/to/folder/folder" ]
|
2021-06-24 18:42:02 +00:00
|
|
|
func directoryDiff(prev, current string) []string {
|
|
|
|
var paths []string
|
|
|
|
|
|
|
|
if prev == "" || current == "" {
|
|
|
|
return paths
|
|
|
|
}
|
|
|
|
|
|
|
|
parent := current
|
|
|
|
for {
|
|
|
|
parent = filepath.Dir(parent)
|
2022-07-10 02:04:50 +00:00
|
|
|
if parent == "/" || parent == prev || strings.HasPrefix(prev+"/", parent+"/") {
|
2021-06-24 18:42:02 +00:00
|
|
|
break
|
|
|
|
}
|
|
|
|
paths = append(paths, parent)
|
|
|
|
}
|
|
|
|
reverse(paths)
|
|
|
|
return paths
|
|
|
|
}
|
|
|
|
|
|
|
|
func reverse(s []string) {
|
|
|
|
for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
|
|
|
|
s[i], s[j] = s[j], s[i]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-04 00:54:52 +00:00
|
|
|
func (d *driver) s3Path(path string) string {
|
2015-01-24 00:46:43 +00:00
|
|
|
return strings.TrimLeft(strings.TrimRight(d.RootDirectory, "/")+path, "/")
|
2014-12-19 17:16:51 +00:00
|
|
|
}
|
2014-10-24 23:37:25 +00:00
|
|
|
|
2015-04-06 23:23:31 +00:00
|
|
|
// S3BucketKey returns the s3 bucket key for the given storage driver path.
|
|
|
|
func (d *Driver) S3BucketKey(path string) string {
|
|
|
|
return d.StorageDriver.(*driver).s3Path(path)
|
|
|
|
}
|
|
|
|
|
2014-12-19 17:16:51 +00:00
|
|
|
func parseError(path string, err error) error {
|
2016-01-22 02:17:53 +00:00
|
|
|
if s3Err, ok := err.(awserr.Error); ok && s3Err.Code() == "NoSuchKey" {
|
2014-12-19 17:16:51 +00:00
|
|
|
return storagedriver.PathNotFoundError{Path: path}
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2014-12-19 17:16:51 +00:00
|
|
|
return err
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
func (d *driver) getEncryptionMode() *string {
|
2016-03-10 00:52:59 +00:00
|
|
|
if !d.Encrypt {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
if d.KeyID == "" {
|
2016-01-22 02:17:53 +00:00
|
|
|
return aws.String("AES256")
|
|
|
|
}
|
2016-03-10 00:52:59 +00:00
|
|
|
return aws.String("aws:kms")
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *driver) getSSEKMSKeyID() *string {
|
|
|
|
if d.KeyID != "" {
|
|
|
|
return aws.String(d.KeyID)
|
|
|
|
}
|
2016-01-22 02:17:53 +00:00
|
|
|
return nil
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
func (d *driver) getContentType() *string {
|
|
|
|
return aws.String("application/octet-stream")
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
func (d *driver) getACL() *string {
|
2016-10-06 00:47:12 +00:00
|
|
|
return aws.String(d.ObjectACL)
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
|
|
|
|
2016-01-22 02:17:53 +00:00
|
|
|
func (d *driver) getStorageClass() *string {
|
2016-09-01 20:52:40 +00:00
|
|
|
if d.StorageClass == noStorageClass {
|
|
|
|
return nil
|
|
|
|
}
|
2016-01-22 02:17:53 +00:00
|
|
|
return aws.String(d.StorageClass)
|
2014-10-24 23:37:25 +00:00
|
|
|
}
|
2015-04-22 22:07:18 +00:00
|
|
|
|
2023-09-22 13:53:47 +00:00
|
|
|
// buffer is a static size bytes buffer.
|
|
|
|
type buffer struct {
|
|
|
|
data []byte
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewBuffer returns a new bytes buffer from driver's memory pool.
|
|
|
|
// The size of the buffer is static and set to params.ChunkSize.
|
|
|
|
func (d *driver) NewBuffer() *buffer {
|
|
|
|
return d.pool.Get().(*buffer)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ReadFrom reads as much data as it can fit in from r without growing its size.
|
|
|
|
// It returns the number of bytes successfully read from r or error.
|
|
|
|
func (b *buffer) ReadFrom(r io.Reader) (offset int64, err error) {
|
|
|
|
for len(b.data) < cap(b.data) && err == nil {
|
|
|
|
var n int
|
|
|
|
n, err = r.Read(b.data[len(b.data):cap(b.data)])
|
|
|
|
offset += int64(n)
|
|
|
|
b.data = b.data[:len(b.data)+n]
|
|
|
|
}
|
|
|
|
// NOTE(milosgajdos): io.ReaderFrom "swallows" io.EOF
|
|
|
|
// See: https://pkg.go.dev/io#ReaderFrom
|
|
|
|
if err == io.EOF {
|
|
|
|
err = nil
|
|
|
|
}
|
|
|
|
return offset, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Cap returns the capacity of the buffer's underlying byte slice.
|
|
|
|
func (b *buffer) Cap() int {
|
|
|
|
return cap(b.data)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Len returns the length of the data in the buffer
|
|
|
|
func (b *buffer) Len() int {
|
|
|
|
return len(b.data)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Clear the buffer data.
|
|
|
|
func (b *buffer) Clear() {
|
|
|
|
b.data = b.data[:0]
|
|
|
|
}
|
|
|
|
|
2016-02-08 22:29:21 +00:00
|
|
|
// writer attempts to upload parts to S3 in a buffered fashion where the last
|
|
|
|
// part is at least as large as the chunksize, so the multipart upload could be
|
|
|
|
// cleanly resumed in the future. This is violated if Close is called after less
|
|
|
|
// than a full chunk is written.
|
|
|
|
type writer struct {
|
2023-10-18 09:34:10 +00:00
|
|
|
ctx context.Context
|
2023-09-22 13:53:47 +00:00
|
|
|
driver *driver
|
|
|
|
key string
|
|
|
|
uploadID string
|
|
|
|
parts []*s3.Part
|
|
|
|
size int64
|
|
|
|
ready *buffer
|
|
|
|
pending *buffer
|
|
|
|
closed bool
|
|
|
|
committed bool
|
|
|
|
cancelled bool
|
2015-04-22 22:07:18 +00:00
|
|
|
}
|
|
|
|
|
2023-10-18 09:34:10 +00:00
|
|
|
func (d *driver) newWriter(ctx context.Context, key, uploadID string, parts []*s3.Part) storagedriver.FileWriter {
|
2016-02-08 22:29:21 +00:00
|
|
|
var size int64
|
|
|
|
for _, part := range parts {
|
|
|
|
size += *part.Size
|
|
|
|
}
|
|
|
|
return &writer{
|
2023-10-18 09:34:10 +00:00
|
|
|
ctx: ctx,
|
2016-02-08 22:29:21 +00:00
|
|
|
driver: d,
|
|
|
|
key: key,
|
|
|
|
uploadID: uploadID,
|
|
|
|
parts: parts,
|
|
|
|
size: size,
|
2023-09-22 13:53:47 +00:00
|
|
|
ready: d.NewBuffer(),
|
|
|
|
pending: d.NewBuffer(),
|
2016-02-08 22:29:21 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-28 14:57:22 +00:00
|
|
|
type completedParts []*s3.CompletedPart
|
|
|
|
|
|
|
|
func (a completedParts) Len() int { return len(a) }
|
|
|
|
func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
|
|
|
func (a completedParts) Less(i, j int) bool { return *a[i].PartNumber < *a[j].PartNumber }
|
|
|
|
|
2016-02-08 22:29:21 +00:00
|
|
|
func (w *writer) Write(p []byte) (int, error) {
|
|
|
|
if w.closed {
|
|
|
|
return 0, fmt.Errorf("already closed")
|
|
|
|
} else if w.committed {
|
|
|
|
return 0, fmt.Errorf("already committed")
|
|
|
|
} else if w.cancelled {
|
|
|
|
return 0, fmt.Errorf("already cancelled")
|
|
|
|
}
|
|
|
|
|
|
|
|
// If the last written part is smaller than minChunkSize, we need to make a
|
|
|
|
// new multipart upload :sadface:
|
|
|
|
if len(w.parts) > 0 && int(*w.parts[len(w.parts)-1].Size) < minChunkSize {
|
2023-09-22 13:53:47 +00:00
|
|
|
completedUploadedParts := make(completedParts, len(w.parts))
|
|
|
|
for i, part := range w.parts {
|
|
|
|
completedUploadedParts[i] = &s3.CompletedPart{
|
2016-02-08 22:29:21 +00:00
|
|
|
ETag: part.ETag,
|
|
|
|
PartNumber: part.PartNumber,
|
2023-09-22 13:53:47 +00:00
|
|
|
}
|
2016-02-08 22:29:21 +00:00
|
|
|
}
|
2016-04-23 10:13:15 +00:00
|
|
|
|
|
|
|
sort.Sort(completedUploadedParts)
|
|
|
|
|
2023-10-18 09:34:10 +00:00
|
|
|
_, err := w.driver.S3.CompleteMultipartUploadWithContext(w.ctx, &s3.CompleteMultipartUploadInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(w.driver.Bucket),
|
|
|
|
Key: aws.String(w.key),
|
|
|
|
UploadId: aws.String(w.uploadID),
|
|
|
|
MultipartUpload: &s3.CompletedMultipartUpload{
|
2016-04-23 10:13:15 +00:00
|
|
|
Parts: completedUploadedParts,
|
2016-02-08 22:29:21 +00:00
|
|
|
},
|
|
|
|
})
|
|
|
|
if err != nil {
|
2023-10-18 09:34:10 +00:00
|
|
|
if _, aErr := w.driver.S3.AbortMultipartUploadWithContext(w.ctx, &s3.AbortMultipartUploadInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(w.driver.Bucket),
|
|
|
|
Key: aws.String(w.key),
|
|
|
|
UploadId: aws.String(w.uploadID),
|
2023-09-22 13:53:47 +00:00
|
|
|
}); aErr != nil {
|
|
|
|
return 0, errors.Join(err, aErr)
|
|
|
|
}
|
2016-02-08 22:29:21 +00:00
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
|
2023-10-18 09:34:10 +00:00
|
|
|
resp, err := w.driver.S3.CreateMultipartUploadWithContext(w.ctx, &s3.CreateMultipartUploadInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(w.driver.Bucket),
|
|
|
|
Key: aws.String(w.key),
|
|
|
|
ContentType: w.driver.getContentType(),
|
|
|
|
ACL: w.driver.getACL(),
|
|
|
|
ServerSideEncryption: w.driver.getEncryptionMode(),
|
|
|
|
StorageClass: w.driver.getStorageClass(),
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
w.uploadID = *resp.UploadId
|
|
|
|
|
|
|
|
// If the entire written file is smaller than minChunkSize, we need to make
|
|
|
|
// a new part from scratch :double sad face:
|
|
|
|
if w.size < minChunkSize {
|
2023-10-18 09:34:10 +00:00
|
|
|
resp, err := w.driver.S3.GetObjectWithContext(w.ctx, &s3.GetObjectInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(w.driver.Bucket),
|
|
|
|
Key: aws.String(w.key),
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
2018-08-06 21:34:15 +00:00
|
|
|
defer resp.Body.Close()
|
2023-09-22 13:53:47 +00:00
|
|
|
|
|
|
|
// reset uploaded parts
|
2016-02-08 22:29:21 +00:00
|
|
|
w.parts = nil
|
2023-09-22 13:53:47 +00:00
|
|
|
w.ready.Clear()
|
|
|
|
|
|
|
|
n, err := w.ready.ReadFrom(resp.Body)
|
2016-02-08 22:29:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
2023-09-22 13:53:47 +00:00
|
|
|
if resp.ContentLength != nil && n < *resp.ContentLength {
|
|
|
|
return 0, io.ErrShortBuffer
|
|
|
|
}
|
2016-02-08 22:29:21 +00:00
|
|
|
} else {
|
|
|
|
// Otherwise we can use the old file as the new first part
|
2023-10-18 09:34:10 +00:00
|
|
|
copyPartResp, err := w.driver.S3.UploadPartCopyWithContext(w.ctx, &s3.UploadPartCopyInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(w.driver.Bucket),
|
|
|
|
CopySource: aws.String(w.driver.Bucket + "/" + w.key),
|
|
|
|
Key: aws.String(w.key),
|
|
|
|
PartNumber: aws.Int64(1),
|
|
|
|
UploadId: resp.UploadId,
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
w.parts = []*s3.Part{
|
|
|
|
{
|
|
|
|
ETag: copyPartResp.CopyPartResult.ETag,
|
|
|
|
PartNumber: aws.Int64(1),
|
|
|
|
Size: aws.Int64(w.size),
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
var n int
|
|
|
|
|
2023-09-22 13:53:47 +00:00
|
|
|
defer func() { w.size += int64(n) }()
|
|
|
|
|
|
|
|
reader := bytes.NewReader(p)
|
|
|
|
|
|
|
|
for reader.Len() > 0 {
|
|
|
|
// NOTE(milosgajdos): we do some seemingly unsafe conversions
|
|
|
|
// from int64 to int in this for loop. These are fine as the
|
|
|
|
// offset returned from buffer.ReadFrom can only ever be
|
|
|
|
// maxChunkSize large which fits in to int. The reason why
|
|
|
|
// we return int64 is to play nice with Go interfaces where
|
|
|
|
// the buffer implements io.ReaderFrom interface.
|
|
|
|
|
|
|
|
// fill up the ready parts buffer
|
|
|
|
offset, err := w.ready.ReadFrom(reader)
|
|
|
|
n += int(offset)
|
|
|
|
if err != nil {
|
|
|
|
return n, err
|
2016-02-08 22:29:21 +00:00
|
|
|
}
|
|
|
|
|
2023-09-22 13:53:47 +00:00
|
|
|
// try filling up the pending parts buffer
|
|
|
|
offset, err = w.pending.ReadFrom(reader)
|
|
|
|
n += int(offset)
|
|
|
|
if err != nil {
|
|
|
|
return n, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// we filled up pending buffer, flush
|
|
|
|
if w.pending.Len() == w.pending.Cap() {
|
|
|
|
if err := w.flush(); err != nil {
|
|
|
|
return n, err
|
2016-02-08 22:29:21 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-09-22 13:53:47 +00:00
|
|
|
|
2016-02-08 22:29:21 +00:00
|
|
|
return n, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (w *writer) Size() int64 {
|
|
|
|
return w.size
|
|
|
|
}
|
|
|
|
func (w *writer) Close() error {
|
|
|
|
if w.closed {
|
|
|
|
return fmt.Errorf("already closed")
|
|
|
|
}
|
|
|
|
w.closed = true
|
2023-09-22 13:53:47 +00:00
|
|
|
|
|
|
|
defer func() {
|
|
|
|
w.ready.Clear()
|
|
|
|
w.driver.pool.Put(w.ready)
|
|
|
|
w.pending.Clear()
|
|
|
|
w.driver.pool.Put(w.pending)
|
|
|
|
}()
|
|
|
|
|
|
|
|
return w.flush()
|
2016-02-08 22:29:21 +00:00
|
|
|
}
|
|
|
|
|
2020-02-21 03:58:17 +00:00
|
|
|
func (w *writer) Cancel(ctx context.Context) error {
|
2016-02-08 22:29:21 +00:00
|
|
|
if w.closed {
|
|
|
|
return fmt.Errorf("already closed")
|
|
|
|
} else if w.committed {
|
|
|
|
return fmt.Errorf("already committed")
|
|
|
|
}
|
|
|
|
w.cancelled = true
|
2023-10-18 09:34:10 +00:00
|
|
|
_, err := w.driver.S3.AbortMultipartUploadWithContext(ctx, &s3.AbortMultipartUploadInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(w.driver.Bucket),
|
|
|
|
Key: aws.String(w.key),
|
|
|
|
UploadId: aws.String(w.uploadID),
|
|
|
|
})
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-10-18 09:34:10 +00:00
|
|
|
func (w *writer) Commit(ctx context.Context) error {
|
2016-02-08 22:29:21 +00:00
|
|
|
if w.closed {
|
|
|
|
return fmt.Errorf("already closed")
|
|
|
|
} else if w.committed {
|
|
|
|
return fmt.Errorf("already committed")
|
|
|
|
} else if w.cancelled {
|
|
|
|
return fmt.Errorf("already cancelled")
|
|
|
|
}
|
2023-09-22 13:53:47 +00:00
|
|
|
|
|
|
|
err := w.flush()
|
2016-02-08 22:29:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-09-22 13:53:47 +00:00
|
|
|
|
2016-02-08 22:29:21 +00:00
|
|
|
w.committed = true
|
2016-04-23 10:13:15 +00:00
|
|
|
|
2023-09-22 13:53:47 +00:00
|
|
|
completedUploadedParts := make(completedParts, len(w.parts))
|
|
|
|
for i, part := range w.parts {
|
|
|
|
completedUploadedParts[i] = &s3.CompletedPart{
|
2016-02-08 22:29:21 +00:00
|
|
|
ETag: part.ETag,
|
|
|
|
PartNumber: part.PartNumber,
|
2023-09-22 13:53:47 +00:00
|
|
|
}
|
2016-02-08 22:29:21 +00:00
|
|
|
}
|
2016-04-23 10:13:15 +00:00
|
|
|
|
2023-09-22 13:53:47 +00:00
|
|
|
// This is an edge case when we are trying to upload an empty file as part of
|
|
|
|
// the MultiPart upload. We get a PUT with Content-Length: 0 and sad things happen.
|
|
|
|
// The result is we are trying to Complete MultipartUpload with an empty list of
|
|
|
|
// completedUploadedParts which will always lead to 400 being returned from S3
|
|
|
|
// See: https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#CompletedMultipartUpload
|
|
|
|
// Solution: we upload the empty i.e. 0 byte part as a single part and then append it
|
2022-10-29 18:42:28 +00:00
|
|
|
// to the completedUploadedParts slice used to complete the Multipart upload.
|
|
|
|
if len(w.parts) == 0 {
|
2023-10-18 09:34:10 +00:00
|
|
|
resp, err := w.driver.S3.UploadPartWithContext(w.ctx, &s3.UploadPartInput{
|
2022-10-29 18:42:28 +00:00
|
|
|
Bucket: aws.String(w.driver.Bucket),
|
|
|
|
Key: aws.String(w.key),
|
|
|
|
PartNumber: aws.Int64(1),
|
|
|
|
UploadId: aws.String(w.uploadID),
|
|
|
|
Body: bytes.NewReader(nil),
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
completedUploadedParts = append(completedUploadedParts, &s3.CompletedPart{
|
|
|
|
ETag: resp.ETag,
|
|
|
|
PartNumber: aws.Int64(1),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2016-04-23 10:13:15 +00:00
|
|
|
sort.Sort(completedUploadedParts)
|
|
|
|
|
2023-10-18 09:34:10 +00:00
|
|
|
_, err = w.driver.S3.CompleteMultipartUploadWithContext(w.ctx, &s3.CompleteMultipartUploadInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(w.driver.Bucket),
|
|
|
|
Key: aws.String(w.key),
|
|
|
|
UploadId: aws.String(w.uploadID),
|
|
|
|
MultipartUpload: &s3.CompletedMultipartUpload{
|
2016-04-23 10:13:15 +00:00
|
|
|
Parts: completedUploadedParts,
|
2016-02-08 22:29:21 +00:00
|
|
|
},
|
|
|
|
})
|
|
|
|
if err != nil {
|
2023-10-18 09:34:10 +00:00
|
|
|
if _, aErr := w.driver.S3.AbortMultipartUploadWithContext(w.ctx, &s3.AbortMultipartUploadInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(w.driver.Bucket),
|
|
|
|
Key: aws.String(w.key),
|
|
|
|
UploadId: aws.String(w.uploadID),
|
2023-09-22 13:53:47 +00:00
|
|
|
}); aErr != nil {
|
|
|
|
return errors.Join(err, aErr)
|
|
|
|
}
|
2016-02-08 22:29:21 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-09-22 13:53:47 +00:00
|
|
|
// flush flushes all buffers to write a part to S3.
|
|
|
|
// flush is only called by Write (with both buffers full) and Close/Commit (always)
|
|
|
|
func (w *writer) flush() error {
|
|
|
|
if w.ready.Len() == 0 && w.pending.Len() == 0 {
|
2016-02-08 22:29:21 +00:00
|
|
|
return nil
|
|
|
|
}
|
2023-09-22 13:53:47 +00:00
|
|
|
|
|
|
|
buf := bytes.NewBuffer(w.ready.data)
|
|
|
|
if w.driver.MultipartCombineSmallPart && (w.pending.Len() > 0 && w.pending.Len() < int(w.driver.ChunkSize)) {
|
|
|
|
if _, err := buf.Write(w.pending.data); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
w.pending.Clear()
|
2016-02-08 22:29:21 +00:00
|
|
|
}
|
|
|
|
|
2023-09-22 13:53:47 +00:00
|
|
|
partSize := buf.Len()
|
2016-02-08 22:29:21 +00:00
|
|
|
partNumber := aws.Int64(int64(len(w.parts) + 1))
|
2023-09-22 13:53:47 +00:00
|
|
|
|
2023-10-18 09:34:10 +00:00
|
|
|
resp, err := w.driver.S3.UploadPartWithContext(w.ctx, &s3.UploadPartInput{
|
2016-02-08 22:29:21 +00:00
|
|
|
Bucket: aws.String(w.driver.Bucket),
|
|
|
|
Key: aws.String(w.key),
|
|
|
|
PartNumber: partNumber,
|
|
|
|
UploadId: aws.String(w.uploadID),
|
2023-09-22 13:53:47 +00:00
|
|
|
Body: bytes.NewReader(buf.Bytes()),
|
2016-02-08 22:29:21 +00:00
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-09-22 13:53:47 +00:00
|
|
|
|
2016-02-08 22:29:21 +00:00
|
|
|
w.parts = append(w.parts, &s3.Part{
|
|
|
|
ETag: resp.ETag,
|
|
|
|
PartNumber: partNumber,
|
2023-09-22 13:53:47 +00:00
|
|
|
Size: aws.Int64(int64(partSize)),
|
2016-02-08 22:29:21 +00:00
|
|
|
})
|
2023-09-22 13:53:47 +00:00
|
|
|
|
|
|
|
// reset the flushed buffer and swap buffers
|
|
|
|
w.ready.Clear()
|
|
|
|
w.ready, w.pending = w.pending, w.ready
|
|
|
|
|
2016-02-08 22:29:21 +00:00
|
|
|
return nil
|
2015-04-22 22:07:18 +00:00
|
|
|
}
|