oracle-object-storage: bring your own encryption keys

backend/oracleobjectstorage/byok.go

@@ -0,0 +1,145 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package oracleobjectstorage
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/objectstorage"
+	"github.com/oracle/oci-go-sdk/v65/objectstorage/transfer"
+)
+
+const (
+	sseDefaultAlgorithm = "AES256"
+)
+
+func getSha256(p []byte) []byte {
+	h := sha256.New()
+	h.Write(p)
+	return h.Sum(nil)
+}
+
+func validateSSECustomerKeyOptions(opt *Options) error {
+	if opt.SSEKMSKeyID != "" && (opt.SSECustomerKeyFile != "" || opt.SSECustomerKey != "") {
+		return errors.New("oos: can't use vault sse_kms_key_id and local sse_customer_key at the same time")
+	}
+	if opt.SSECustomerKey != "" && opt.SSECustomerKeyFile != "" {
+		return errors.New("oos: can't use sse_customer_key and sse_customer_key_file at the same time")
+	}
+	if opt.SSEKMSKeyID != "" {
+		return nil
+	}
+	err := populateSSECustomerKeys(opt)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func populateSSECustomerKeys(opt *Options) error {
+	if opt.SSECustomerKeyFile != "" {
+		// Reads the base64-encoded AES key data from the specified file and computes its SHA256 checksum
+		data, err := os.ReadFile(expandPath(opt.SSECustomerKeyFile))
+		if err != nil {
+			return fmt.Errorf("oos: error reading sse_customer_key_file: %v", err)
+		}
+		opt.SSECustomerKey = strings.TrimSpace(string(data))
+	}
+	if opt.SSECustomerKey != "" {
+		decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKey)
+		if err != nil {
+			return fmt.Errorf("oos: Could not decode sse_customer_key_file: %w", err)
+		}
+		sha256Checksum := base64.StdEncoding.EncodeToString(getSha256(decoded))
+		if opt.SSECustomerKeySha256 == "" {
+			opt.SSECustomerKeySha256 = sha256Checksum
+		} else {
+			if opt.SSECustomerKeySha256 != sha256Checksum {
+				return fmt.Errorf("the computed SHA256 checksum "+
+					"(%v) of the key doesn't match the config entry sse_customer_key_sha256=(%v)",
+					sha256Checksum, opt.SSECustomerKeySha256)
+			}
+		}
+		if opt.SSECustomerAlgorithm == "" {
+			opt.SSECustomerAlgorithm = sseDefaultAlgorithm
+		}
+	}
+	return nil
+}
+
+// https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/usingyourencryptionkeys.htm
+func useBYOKPutObject(fs *Fs, request *objectstorage.PutObjectRequest) {
+	if fs.opt.SSEKMSKeyID != "" {
+		request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
+	}
+	if fs.opt.SSECustomerAlgorithm != "" {
+		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+	}
+	if fs.opt.SSECustomerKey != "" {
+		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+	}
+	if fs.opt.SSECustomerKeySha256 != "" {
+		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+	}
+}
+
+func useBYOKHeadObject(fs *Fs, request *objectstorage.HeadObjectRequest) {
+	if fs.opt.SSECustomerAlgorithm != "" {
+		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+	}
+	if fs.opt.SSECustomerKey != "" {
+		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+	}
+	if fs.opt.SSECustomerKeySha256 != "" {
+		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+	}
+}
+
+func useBYOKGetObject(fs *Fs, request *objectstorage.GetObjectRequest) {
+	if fs.opt.SSECustomerAlgorithm != "" {
+		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+	}
+	if fs.opt.SSECustomerKey != "" {
+		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+	}
+	if fs.opt.SSECustomerKeySha256 != "" {
+		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+	}
+}
+
+func useBYOKCopyObject(fs *Fs, request *objectstorage.CopyObjectRequest) {
+	if fs.opt.SSEKMSKeyID != "" {
+		request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
+	}
+	if fs.opt.SSECustomerAlgorithm != "" {
+		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+	}
+	if fs.opt.SSECustomerKey != "" {
+		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+	}
+	if fs.opt.SSECustomerKeySha256 != "" {
+		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+	}
+}
+
+func useBYOKUpload(fs *Fs, request *transfer.UploadRequest) {
+	if fs.opt.SSEKMSKeyID != "" {
+		request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
+	}
+	if fs.opt.SSECustomerAlgorithm != "" {
+		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+	}
+	if fs.opt.SSECustomerKey != "" {
+		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+	}
+	if fs.opt.SSECustomerKeySha256 != "" {
+		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+	}
+}

backend/oracleobjectstorage/client.go

@@ -9,6 +9,8 @@ import (
 	"errors"
 	"net/http"
 	"os"
+	"path"
+	"strings"
 
 	"github.com/oracle/oci-go-sdk/v65/common"
 	"github.com/oracle/oci-go-sdk/v65/common/auth"
@@ -18,15 +20,33 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 )
 
+func expandPath(filepath string) (expandedPath string) {
+	if filepath == "" {
+		return filepath
+	}
+	cleanedPath := path.Clean(filepath)
+	expandedPath = cleanedPath
+	if strings.HasPrefix(cleanedPath, "~") {
+		rest := cleanedPath[2:]
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return expandedPath
+		}
+		expandedPath = path.Join(home, rest)
+	}
+	return
+}
+
 func getConfigurationProvider(opt *Options) (common.ConfigurationProvider, error) {
 	switch opt.Provider {
 	case instancePrincipal:
 		return auth.InstancePrincipalConfigurationProvider()
 	case userPrincipal:
-		if opt.ConfigFile != "" && !fileExists(opt.ConfigFile) {
+		expandConfigFilePath := expandPath(opt.ConfigFile)
-			fs.Errorf(userPrincipal, "oci config file doesn't exist at %v", opt.ConfigFile)
+		if expandConfigFilePath != "" && !fileExists(expandConfigFilePath) {
+			fs.Errorf(userPrincipal, "oci config file doesn't exist at %v", expandConfigFilePath)
 		}
-		return common.CustomProfileConfigProvider(opt.ConfigFile, opt.ConfigProfile), nil
+		return common.CustomProfileConfigProvider(expandConfigFilePath, opt.ConfigProfile), nil
 	case resourcePrincipal:
 		return auth.ResourcePrincipalConfigurationProvider()
 	case noAuth:

backend/oracleobjectstorage/copy.go

@@ -74,6 +74,7 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object) (err erro
 		BucketName:        common.String(srcBucket),
 		CopyObjectDetails: copyObjectDetails,
 	}
+	useBYOKCopyObject(f, &req)
 	var resp objectstorage.CopyObjectResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CopyObject(ctx, req)

backend/oracleobjectstorage/object.go

@@ -87,6 +87,7 @@ func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObject
 		BucketName:    common.String(bucketName),
 		ObjectName:    common.String(objectPath),
 	}
+	useBYOKHeadObject(o.fs, &req)
 	var response objectstorage.HeadObjectResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
@@ -99,6 +100,7 @@ func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObject
 				return nil, fs.ErrorObjectNotFound
 			}
 		}
+		fs.Errorf(o, "Failed to head object: %v", err)
 		return nil, err
 	}
 	o.fs.cache.MarkOK(bucketName)
@@ -331,7 +333,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 		ObjectName:    common.String(bucketPath),
 	}
 	o.applyGetObjectOptions(&req, options...)
-
+	useBYOKGetObject(o.fs, &req)
 	var resp objectstorage.GetObjectResponse
 	err := o.fs.pacer.Call(func() (bool, error) {
 		var err error
@@ -433,6 +435,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			uploadRequest.StorageTier = storageTier
 		}
 		o.applyMultiPutOptions(&uploadRequest, options...)
+		useBYOKUpload(o.fs, &uploadRequest)
 		uploadStreamRequest := transfer.UploadStreamRequest{
 			UploadRequest: uploadRequest,
 			StreamReader:  in,
@@ -506,8 +509,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			req.StorageTier = storageTier
 		}
 		o.applyPutOptions(&req, options...)
+		useBYOKPutObject(o.fs, &req)
+		var resp objectstorage.PutObjectResponse
 		err = o.fs.pacer.Call(func() (bool, error) {
-			resp, err := o.fs.srv.PutObject(ctx, req)
+			resp, err = o.fs.srv.PutObject(ctx, req)
 			return shouldRetry(ctx, resp.HTTPResponse(), err)
 		})
 		if err != nil {

backend/oracleobjectstorage/options.go

@@ -45,23 +45,28 @@ https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfromins
 
 // Options defines the configuration for this backend
 type Options struct {
-	Provider          string               `config:"provider"`
+	Provider             string               `config:"provider"`
-	Compartment       string               `config:"compartment"`
+	Compartment          string               `config:"compartment"`
-	Namespace         string               `config:"namespace"`
+	Namespace            string               `config:"namespace"`
-	Region            string               `config:"region"`
+	Region               string               `config:"region"`
-	Endpoint          string               `config:"endpoint"`
+	Endpoint             string               `config:"endpoint"`
-	Enc               encoder.MultiEncoder `config:"encoding"`
+	Enc                  encoder.MultiEncoder `config:"encoding"`
-	ConfigFile        string               `config:"config_file"`
+	ConfigFile           string               `config:"config_file"`
-	ConfigProfile     string               `config:"config_profile"`
+	ConfigProfile        string               `config:"config_profile"`
-	UploadCutoff      fs.SizeSuffix        `config:"upload_cutoff"`
+	UploadCutoff         fs.SizeSuffix        `config:"upload_cutoff"`
-	ChunkSize         fs.SizeSuffix        `config:"chunk_size"`
+	ChunkSize            fs.SizeSuffix        `config:"chunk_size"`
-	UploadConcurrency int                  `config:"upload_concurrency"`
+	UploadConcurrency    int                  `config:"upload_concurrency"`
-	DisableChecksum   bool                 `config:"disable_checksum"`
+	DisableChecksum      bool                 `config:"disable_checksum"`
-	CopyCutoff        fs.SizeSuffix        `config:"copy_cutoff"`
+	CopyCutoff           fs.SizeSuffix        `config:"copy_cutoff"`
-	CopyTimeout       fs.Duration          `config:"copy_timeout"`
+	CopyTimeout          fs.Duration          `config:"copy_timeout"`
-	StorageTier       string               `config:"storage_tier"`
+	StorageTier          string               `config:"storage_tier"`
-	LeavePartsOnError bool                 `config:"leave_parts_on_error"`
+	LeavePartsOnError    bool                 `config:"leave_parts_on_error"`
-	NoCheckBucket     bool                 `config:"no_check_bucket"`
+	NoCheckBucket        bool                 `config:"no_check_bucket"`
+	SSEKMSKeyID          string               `config:"sse_kms_key_id"`
+	SSECustomerAlgorithm string               `config:"sse_customer_algorithm"`
+	SSECustomerKey       string               `config:"sse_customer_key"`
+	SSECustomerKeyFile   string               `config:"sse_customer_key_file"`
+	SSECustomerKeySha256 string               `config:"sse_customer_key_sha256"`
 }
 
 func newOptions() []fs.Option {
@@ -252,5 +257,59 @@ creation permissions.
 `,
 		Default:  false,
 		Advanced: true,
+	}, {
+		Name: "sse_customer_key_file",
+		Help: `To use SSE-C, a file containing the base64-encoded string of the AES-256 encryption key associated
+with the object. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.'`,
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "",
+			Help:  "None",
+		}},
+	}, {
+		Name: "sse_customer_key",
+		Help: `To use SSE-C, the optional header that specifies the base64-encoded 256-bit encryption key to use to
+encrypt or  decrypt the data. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is
+needed. For more information, see Using Your Own Keys for Server-Side Encryption 
+(https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm)`,
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "",
+			Help:  "None",
+		}},
+	}, {
+		Name: "sse_customer_key_sha256",
+		Help: `If using SSE-C, The optional header that specifies the base64-encoded SHA256 hash of the encryption
+key. This value is used to check the integrity of the encryption key. see Using Your Own Keys for 
+Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`,
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "",
+			Help:  "None",
+		}},
+	}, {
+		Name: "sse_kms_key_id",
+		Help: `if using using your own master key in vault, this header specifies the 
+OCID (https://docs.cloud.oracle.com/Content/General/Concepts/identifiers.htm) of a master encryption key used to call
+the Key Management service to generate a data encryption key or to encrypt or decrypt a data encryption key.
+Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.`,
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "",
+			Help:  "None",
+		}},
+	}, {
+		Name: "sse_customer_algorithm",
+		Help: `If using SSE-C, the optional header that specifies "AES256" as the encryption algorithm.
+Object Storage supports "AES256" as the encryption algorithm. For more information, see
+Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`,
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "",
+			Help:  "None",
+		}, {
+			Value: sseDefaultAlgorithm,
+			Help:  sseDefaultAlgorithm,
+		}},
 	}}
 }

backend/oracleobjectstorage/oracleobjectstorage.go

@@ -59,6 +59,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, err
 	}
+	err = validateSSECustomerKeyOptions(opt)
+	if err != nil {
+		return nil, err
+	}
 	ci := fs.GetConfig(ctx)
 	objectStorageClient, err := newObjectStorageClient(ctx, opt)
 	if err != nil {

docs/content/oracleobjectstorage.md

@@ -561,6 +561,89 @@ Properties:
 - Type:        bool
 - Default:     false
 
+#### --oos-sse-customer-key-file
+
+To use SSE-C, a file containing the base64-encoded string of the AES-256 encryption key associated
+with the object. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.'
+
+Properties:
+
+- Config:      sse_customer_key_file
+- Env Var:     RCLONE_OOS_SSE_CUSTOMER_KEY_FILE
+- Type:        string
+- Required:    false
+- Examples:
+    - ""
+        - None
+
+#### --oos-sse-customer-key
+
+To use SSE-C, the optional header that specifies the base64-encoded 256-bit encryption key to use to
+encrypt or  decrypt the data. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is
+needed. For more information, see Using Your Own Keys for Server-Side Encryption 
+(https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm)
+
+Properties:
+
+- Config:      sse_customer_key
+- Env Var:     RCLONE_OOS_SSE_CUSTOMER_KEY
+- Type:        string
+- Required:    false
+- Examples:
+    - ""
+        - None
+
+#### --oos-sse-customer-key-sha256
+
+If using SSE-C, The optional header that specifies the base64-encoded SHA256 hash of the encryption
+key. This value is used to check the integrity of the encryption key. see Using Your Own Keys for 
+Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).
+
+Properties:
+
+- Config:      sse_customer_key_sha256
+- Env Var:     RCLONE_OOS_SSE_CUSTOMER_KEY_SHA256
+- Type:        string
+- Required:    false
+- Examples:
+    - ""
+        - None
+
+#### --oos-sse-kms-key-id
+
+if using using your own master key in vault, this header specifies the 
+OCID (https://docs.cloud.oracle.com/Content/General/Concepts/identifiers.htm) of a master encryption key used to call
+the Key Management service to generate a data encryption key or to encrypt or decrypt a data encryption key.
+Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.
+
+Properties:
+
+- Config:      sse_kms_key_id
+- Env Var:     RCLONE_OOS_SSE_KMS_KEY_ID
+- Type:        string
+- Required:    false
+- Examples:
+    - ""
+        - None
+
+#### --oos-sse-customer-algorithm
+
+If using SSE-C, the optional header that specifies "AES256" as the encryption algorithm.
+Object Storage supports "AES256" as the encryption algorithm. For more information, see
+Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).
+
+Properties:
+
+- Config:      sse_customer_algorithm
+- Env Var:     RCLONE_OOS_SSE_CUSTOMER_ALGORITHM
+- Type:        string
+- Required:    false
+- Examples:
+    - ""
+        - None
+    - "AES256"
+        - AES256
+
 ## Backend commands
 
 Here are the commands specific to the oracleobjectstorage backend.

go.mod

@@ -41,7 +41,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1
 	github.com/ncw/swift/v2 v2.0.1
-	github.com/oracle/oci-go-sdk/v65 v65.28.0
+	github.com/oracle/oci-go-sdk/v65 v65.30.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6
 	github.com/pmezard/go-difflib v1.0.0

go.sum

@@ -358,8 +358,8 @@ github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E=
-github.com/oracle/oci-go-sdk/v65 v65.28.0 h1:FgBWmHzT1y9H9tRQYOiX22Z8ec5p6/M5sSYRGcbWAno=
+github.com/oracle/oci-go-sdk/v65 v65.30.0 h1:cP1IXZpJ0dxfDjFBulQm5YjA2pUjE83dyDinIp86rv0=
-github.com/oracle/oci-go-sdk/v65 v65.28.0/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE=
+github.com/oracle/oci-go-sdk/v65 v65.30.0/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU=