rclone/backend/netstorage/netstorage.go

1285 lines
40 KiB
Go
Raw Normal View History

// Package netstorage provides an interface to Akamai NetStorage API
package netstorage
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/xml"
"errors"
"fmt"
gohash "hash"
"io"
"math/rand"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
)
// Constants
const (
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
)
func init() {
fsi := &fs.RegInfo{
Name: "netstorage",
Description: "Akamai NetStorage",
NewFs: NewFs,
CommandHelp: commandHelp,
Options: []fs.Option{{
Name: "protocol",
Help: `Select between HTTP or HTTPS protocol.
Most users should choose HTTPS, which is the default.
HTTP is provided primarily for debugging purposes.`,
Examples: []fs.OptionExample{{
Value: "http",
Help: "HTTP protocol",
}, {
Value: "https",
Help: "HTTPS protocol",
}},
Default: "https",
Advanced: true,
}, {
Name: "host",
Help: `Domain+path of NetStorage host to connect to.
Format should be ` + "`<domain>/<internal folders>`",
Required: true,
Sensitive: true,
}, {
Name: "account",
Help: "Set the NetStorage account name",
Required: true,
Sensitive: true,
}, {
Name: "secret",
Help: `Set the NetStorage account secret/G2O key for authentication.
Please choose the 'y' option to set your own password then enter your secret.`,
IsPassword: true,
Required: true,
}},
}
fs.Register(fsi)
}
var commandHelp = []fs.CommandHelp{{
Name: "du",
Short: "Return disk usage information for a specified directory",
Long: `The usage information returned, includes the targeted directory as well as all
files stored in any sub-directories that may exist.`,
}, {
Name: "symlink",
Short: "You can create a symbolic link in ObjectStore with the symlink action.",
Long: `The desired path location (including applicable sub-directories) ending in
the object that will be the target of the symlink (for example, /links/mylink).
Include the file extension for the object, if applicable.
` + "`rclone backend symlink <src> <path>`",
},
}
// Options defines the configuration for this backend
type Options struct {
Endpoint string `config:"host"`
Account string `config:"account"`
Secret string `config:"secret"`
Protocol string `config:"protocol"`
}
// Fs stores the interface to the remote HTTP files
type Fs struct {
name string
root string
features *fs.Features // optional features
opt Options // options for this backend
endpointURL string // endpoint as a string
srv *rest.Client // the connection to the Netstorage server
pacer *fs.Pacer // to pace the API calls
filetype string // dir, file or symlink
dirscreated map[string]bool // if implicit dir has been created already
dirscreatedMutex sync.Mutex // mutex to protect dirscreated
statcache map[string][]File // cache successful stat requests
statcacheMutex sync.RWMutex // RWMutex to protect statcache
}
// Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)
type Object struct {
fs *Fs
filetype string // dir, file or symlink
remote string // remote path
size int64 // size of the object in bytes
modTime int64 // modification time of the object
md5sum string // md5sum of the object
fullURL string // full path URL
target string // symlink target when filetype is symlink
}
//------------------------------------------------------------------------------
// Stat is an object which holds the information of the stat element of the response xml
type Stat struct {
XMLName xml.Name `xml:"stat"`
Files []File `xml:"file"`
Directory string `xml:"directory,attr"`
}
// File is an object which holds the information of the file element of the response xml
type File struct {
XMLName xml.Name `xml:"file"`
Type string `xml:"type,attr"`
Name string `xml:"name,attr"`
NameBase64 string `xml:"name_base64,attr"`
Size int64 `xml:"size,attr"`
Md5 string `xml:"md5,attr"`
Mtime int64 `xml:"mtime,attr"`
Bytes int64 `xml:"bytes,attr"`
Files int64 `xml:"files,attr"`
Target string `xml:"target,attr"`
}
// List is an object which holds the information of the list element of the response xml
type List struct {
XMLName xml.Name `xml:"list"`
Files []File `xml:"file"`
Resume ListResume `xml:"resume"`
}
// ListResume represents the resume xml element of the list
type ListResume struct {
XMLName xml.Name `xml:"resume"`
Start string `xml:"start,attr"`
}
// Du represents the du xml element of the response
type Du struct {
XMLName xml.Name `xml:"du"`
Directory string `xml:"directory,attr"`
Duinfo DuInfo `xml:"du-info"`
}
// DuInfo represents the du-info xml element of the response
type DuInfo struct {
XMLName xml.Name `xml:"du-info"`
Files int64 `xml:"files,attr"`
Bytes int64 `xml:"bytes,attr"`
}
// GetName returns a normalized name of the Stat item
func (s Stat) GetName() xml.Name {
return s.XMLName
}
// GetName returns a normalized name of the List item
func (l List) GetName() xml.Name {
return l.XMLName
}
// GetName returns a normalized name of the Du item
func (d Du) GetName() xml.Name {
return d.XMLName
}
//------------------------------------------------------------------------------
// NewFs creates a new Fs object from the name and root. It connects to
// the host specified in the config file.
//
// If root refers to an existing object, then it should return an Fs which
// points to the parent of that object and ErrorIsFile.
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return nil, err
}
// The base URL (endPoint is protocol + :// + domain/internal folder
opt.Endpoint = opt.Protocol + "://" + opt.Endpoint
fs.Debugf(nil, "NetStorage NewFS endpoint %q", opt.Endpoint)
if !strings.HasSuffix(opt.Endpoint, "/") {
opt.Endpoint += "/"
}
// Decrypt credentials, even though it is hard to eyedrop the hex string, it adds an extra piece of mind
opt.Secret = obscure.MustReveal(opt.Secret)
// Parse the endpoint and stick the root onto it
base, err := url.Parse(opt.Endpoint)
if err != nil {
return nil, fmt.Errorf("couldn't parse URL %q: %w", opt.Endpoint, err)
}
u, err := rest.URLJoin(base, rest.URLPathEscape(root))
if err != nil {
return nil, fmt.Errorf("couldn't join URL %q and %q: %w", base.String(), root, err)
}
client := fshttp.NewClient(ctx)
f := &Fs{
name: name,
root: root,
opt: *opt,
endpointURL: u.String(),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
dirscreated: make(map[string]bool),
statcache: make(map[string][]File),
}
f.srv = rest.NewClient(client)
f.srv.SetSigner(f.getAuth)
f.features = (&fs.Features{
CanHaveEmptyDirectories: true,
}).Fill(ctx, f)
err = f.initFs(ctx, "")
switch err {
case nil:
// Object is the directory
return f, nil
case fs.ErrorObjectNotFound:
return f, nil
case fs.ErrorIsFile:
// Correct root if definitely pointing to a file
f.root = path.Dir(f.root)
if f.root == "." || f.root == "/" {
f.root = ""
}
// Fs points to the parent directory
return f, err
default:
return nil, err
}
}
// Command the backend to run a named commands: du and symlink
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
switch name {
case "du":
// No arg parsing needed, the path is passed in the fs
return f.netStorageDuRequest(ctx)
case "symlink":
dst := ""
if len(arg) > 0 {
dst = arg[0]
} else {
return nil, errors.New("NetStorage symlink command: need argument for target")
}
// Strip off the leading slash added by NewFs on object not found
URL := strings.TrimSuffix(f.url(""), "/")
return f.netStorageSymlinkRequest(ctx, URL, dst, nil)
default:
return nil, fs.ErrorCommandNotFound
}
}
// Name returns the configured name of the file system
func (f *Fs) Name() string {
return f.name
}
// Root returns the root for the filesystem
func (f *Fs) Root() string {
return f.root
}
// String returns the URL for the filesystem
func (f *Fs) String() string {
return f.endpointURL
}
// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
return f.features
}
// Precision return the precision of this Fs
func (f *Fs) Precision() time.Duration {
return time.Second
}
// NewObject creates a new remote http file object
// NewObject finds the Object at remote
// If it can't be found returns fs.ErrorObjectNotFound
// If it isn't a file, then it returns fs.ErrorIsDir
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
URL := f.url(remote)
files, err := f.netStorageStatRequest(ctx, URL, false)
if err != nil {
return nil, err
}
if files == nil {
fs.Errorf(nil, "Stat for %q has empty files", URL)
return nil, fs.ErrorObjectNotFound
}
file := files[0]
switch file.Type {
case
"file",
"symlink":
return f.newObjectWithInfo(remote, &file)
case "dir":
return nil, fs.ErrorIsDir
default:
return nil, fmt.Errorf("object of an unsupported type %s for %q: %w", file.Type, URL, err)
}
}
// initFs initializes Fs based on the stat reply
func (f *Fs) initFs(ctx context.Context, dir string) error {
// Path must end with the slash, so the join later will work correctly
defer func() {
if !strings.HasSuffix(f.endpointURL, "/") {
f.endpointURL += "/"
}
}()
URL := f.url(dir)
files, err := f.netStorageStatRequest(ctx, URL, true)
if err == fs.ErrorObjectNotFound || files == nil {
return fs.ErrorObjectNotFound
}
if err != nil {
return err
}
f.filetype = files[0].Type
switch f.filetype {
case "dir":
// This directory is known to exist, adding to explicit directories
f.setDirscreated(URL)
return nil
case
"file",
"symlink":
// Fs should point to the parent of that object and return ErrorIsFile
lastindex := strings.LastIndex(f.endpointURL, "/")
if lastindex != -1 {
f.endpointURL = f.endpointURL[0 : lastindex+1]
} else {
fs.Errorf(nil, "Remote URL %q unexpectedly does not include the slash", f.endpointURL)
}
return fs.ErrorIsFile
default:
err = fmt.Errorf("unsupported object type %s for %q: %w", f.filetype, URL, err)
f.filetype = ""
return err
}
}
// url joins the remote onto the endpoint URL
func (f *Fs) url(remote string) string {
if remote == "" {
return f.endpointURL
}
pathescapeURL := rest.URLPathEscape(remote)
// Strip off initial "./" from the path, which can be added by path escape function following the RFC 3986 4.2
// (a segment must be preceded by a dot-segment (e.g., "./this:that") to make a relative-path reference).
pathescapeURL = strings.TrimPrefix(pathescapeURL, "./")
// Cannot use rest.URLJoin() here because NetStorage is an object storage and allows to have a "."
// directory name, which will be eliminated by the join function.
return f.endpointURL + pathescapeURL
}
// getFileName returns the file name if present, otherwise decoded name_base64
// if present, otherwise an empty string
func (f *Fs) getFileName(file *File) string {
if file.Name != "" {
return file.Name
}
if file.NameBase64 != "" {
decoded, err := base64.StdEncoding.DecodeString(file.NameBase64)
if err == nil {
return string(decoded)
}
fs.Errorf(nil, "Failed to base64 decode object %s: %v", file.NameBase64, err)
}
return ""
}
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
if f.filetype == "" {
// This happens in two scenarios.
// 1. NewFs is done on a nonexistent object, then later rclone attempts to List/ListR this NewFs.
// 2. List/ListR is called from the context of test_all and not the regular rclone binary.
err := f.initFs(ctx, dir)
if err != nil {
if err == fs.ErrorObjectNotFound {
return nil, fs.ErrorDirNotFound
}
return nil, err
}
}
URL := f.url(dir)
files, err := f.netStorageDirRequest(ctx, dir, URL)
if err != nil {
return nil, err
}
if dir != "" && !strings.HasSuffix(dir, "/") {
dir += "/"
}
for _, item := range files {
name := dir + f.getFileName(&item)
switch item.Type {
case "dir":
when := time.Unix(item.Mtime, 0)
entry := fs.NewDir(name, when).SetSize(item.Bytes).SetItems(item.Files)
entries = append(entries, entry)
case "file":
if entry, _ := f.newObjectWithInfo(name, &item); entry != nil {
entries = append(entries, entry)
}
case "symlink":
var entry fs.Object
// Add .rclonelink suffix to allow local backend code to convert to a symlink.
// In case both .rclonelink file AND symlink file exists, the first will be used.
if entry, _ = f.newObjectWithInfo(name+".rclonelink", &item); entry != nil {
fs.Infof(nil, "Converting a symlink to the rclonelink %s target %s", entry.Remote(), item.Target)
entries = append(entries, entry)
}
default:
fs.Logf(nil, "Ignoring unsupported object type %s for %q path", item.Type, name)
}
}
return entries, nil
}
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
if f.filetype == "" {
// This happens in two scenarios.
// 1. NewFs is done on a nonexistent object, then later rclone attempts to List/ListR this NewFs.
// 2. List/ListR is called from the context of test_all and not the regular rclone binary.
err := f.initFs(ctx, dir)
if err != nil {
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
}
return err
}
}
if !strings.HasSuffix(dir, "/") && dir != "" {
dir += "/"
}
URL := f.url(dir)
u, err := url.Parse(URL)
if err != nil {
fs.Errorf(nil, "Unable to parse URL %q: %v", URL, err)
return fs.ErrorDirNotFound
}
list := walk.NewListRHelper(callback)
for resumeStart := u.Path; resumeStart != ""; {
var files []File
files, resumeStart, err = f.netStorageListRequest(ctx, URL, u.Path)
if err != nil {
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
}
return err
}
for _, item := range files {
name := f.getFileName(&item)
// List output includes full paths starting from [CP Code]/
path := strings.TrimPrefix("/"+name, u.Path)
if path == "" {
// Skip the starting directory itself
continue
}
switch item.Type {
case "dir":
when := time.Unix(item.Mtime, 0)
entry := fs.NewDir(dir+strings.TrimSuffix(path, "/"), when)
if err := list.Add(entry); err != nil {
return err
}
case "file":
if entry, _ := f.newObjectWithInfo(dir+path, &item); entry != nil {
if err := list.Add(entry); err != nil {
return err
}
}
case "symlink":
// Add .rclonelink suffix to allow local backend code to convert to a symlink.
// In case both .rclonelink file AND symlink file exists, the first will be used.
if entry, _ := f.newObjectWithInfo(dir+path+".rclonelink", &item); entry != nil {
fs.Infof(nil, "Converting a symlink to the rclonelink %s for target %s", entry.Remote(), item.Target)
if err := list.Add(entry); err != nil {
return err
}
}
default:
fs.Logf(nil, "Ignoring unsupported object type %s for %s path", item.Type, name)
}
}
if resumeStart != "" {
// Perform subsequent list action call, construct the
// URL where the previous request finished
u, err := url.Parse(f.endpointURL)
if err != nil {
fs.Errorf(nil, "Unable to parse URL %q: %v", f.endpointURL, err)
return fs.ErrorDirNotFound
}
resumeURL, err := rest.URLJoin(u, rest.URLPathEscape(resumeStart))
if err != nil {
fs.Errorf(nil, "Unable to join URL %q for resumeStart %s: %v", f.endpointURL, resumeStart, err)
return fs.ErrorDirNotFound
}
URL = resumeURL.String()
}
}
return list.Flush()
}
// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
err := f.implicitCheck(ctx, src.Remote(), true)
if err != nil {
return nil, err
}
// Barebones object will get filled in Update
o := &Object{
fs: f,
remote: src.Remote(),
fullURL: f.url(src.Remote()),
}
// We pass through the Update's error
err = o.Update(ctx, in, src, options...)
if err != nil {
return nil, err
}
return o, nil
}
// implicitCheck prevents implicit dir creation by doing mkdir from base up to current dir,
// does NOT check if these dirs created conflict with existing dirs/files so can result in dupe
func (f *Fs) implicitCheck(ctx context.Context, remote string, isfile bool) error {
// Find base (URL including the CPCODE path) and root (what follows after that)
URL := f.url(remote)
u, err := url.Parse(URL)
if err != nil {
fs.Errorf(nil, "Unable to parse URL %q while implicit checking directory: %v", URL, err)
return err
}
startPos := 0
if strings.HasPrefix(u.Path, "/") {
startPos = 1
}
pos := strings.Index(u.Path[startPos:], "/")
if pos == -1 {
fs.Errorf(nil, "URL %q unexpectedly does not include the slash in the CPCODE path", URL)
return nil
}
root := rest.URLPathEscape(u.Path[startPos+pos+1:])
u.Path = u.Path[:startPos+pos]
base := u.String()
if !strings.HasSuffix(base, "/") {
base += "/"
}
if isfile {
// Get the base name of root
lastindex := strings.LastIndex(root, "/")
if lastindex == -1 {
// We are at the level of CPCODE path
return nil
}
root = root[0 : lastindex+1]
}
// We make sure root always has "/" at the end
if !strings.HasSuffix(root, "/") {
root += "/"
}
for root != "" {
frontindex := strings.Index(root, "/")
if frontindex == -1 {
return nil
}
frontdir := root[0 : frontindex+1]
root = root[frontindex+1:]
base += frontdir
if !f.testAndSetDirscreated(base) {
fs.Infof(nil, "Implicitly create directory %s", base)
err := f.netStorageMkdirRequest(ctx, base)
if err != nil {
fs.Errorf("Mkdir request in implicit check failed for base %s: %v", base, err)
return err
}
}
}
return nil
}
// Purge all files in the directory specified.
// NetStorage quick-delete is disabled by default AND not instantaneous.
// Returns fs.ErrorCantPurge when quick-delete fails.
func (f *Fs) Purge(ctx context.Context, dir string) error {
URL := f.url(dir)
const actionHeader = "version=1&action=quick-delete&quick-delete=imreallyreallysure"
if _, err := f.callBackend(ctx, URL, "POST", actionHeader, true, nil, nil); err != nil {
fs.Logf(nil, "Purge using quick-delete failed, fallback on recursive delete: %v", err)
return fs.ErrorCantPurge
}
fs.Logf(nil, "Purge using quick-delete has been queued, you may not see immediate changes")
return nil
}
// PutStream uploads to the remote path with the modTime given of indeterminate size
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
// Pass through error from Put
return f.Put(ctx, in, src, options...)
}
// Fs is the filesystem this remote http file object is located within
func (o *Object) Fs() fs.Info {
return o.fs
}
// String returns the URL to the remote HTTP file
func (o *Object) String() string {
if o == nil {
return "<nil>"
}
return o.remote
}
// Remote the name of the remote HTTP file, relative to the fs root
func (o *Object) Remote() string {
return o.remote
}
// Hash returns the Md5sum of an object returning a lowercase hex string
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != hash.MD5 {
return "", hash.ErrUnsupported
}
return o.md5sum, nil
}
// Size returns the size in bytes of the remote http file
func (o *Object) Size() int64 {
return o.size
}
// Md5Sum returns the md5 of the object
func (o *Object) Md5Sum() string {
return o.md5sum
}
// ModTime returns the modification time of the object
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time {
return time.Unix(o.modTime, 0)
}
// SetModTime sets the modification and access time to the specified time
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
URL := o.fullURL
when := strconv.FormatInt(modTime.Unix(), 10)
actionHeader := "version=1&action=mtime&mtime=" + when
if _, err := o.fs.callBackend(ctx, URL, "POST", actionHeader, true, nil, nil); err != nil {
fs.Debugf(nil, "NetStorage action mtime failed for %q: %v", URL, err)
return err
}
o.fs.deleteStatCache(URL)
o.modTime = modTime.Unix()
return nil
}
// Storable returns whether this object is storable
func (o *Object) Storable() bool {
return true
}
// Open an object for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
return o.netStorageDownloadRequest(ctx, options)
}
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
return hash.Set(hash.MD5)
}
// Mkdir makes the root directory of the Fs object
// Shouldn't return an error if it already exists
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// ImplicitCheck will mkdir from base up to dir, if not already in dirscreated
return f.implicitCheck(ctx, dir, false)
}
// Remove an object
func (o *Object) Remove(ctx context.Context) error {
return o.netStorageDeleteRequest(ctx)
}
// Rmdir removes the root directory of the Fs object
// Return an error if it doesn't exist or isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
return f.netStorageRmdirRequest(ctx, dir)
}
// Update netstorage with the object
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
o.size = src.Size()
o.modTime = src.ModTime(ctx).Unix()
// Don't do md5 check because that's done by server
o.md5sum = ""
err := o.netStorageUploadRequest(ctx, in, src)
// We return an object updated with source stats,
// we don't refetch the obj after upload
if err != nil {
return err
}
return nil
}
// newObjectWithInfo creates an fs.Object for any netstorage.File or symlink.
// If it can't be found it returns the error fs.ErrorObjectNotFound.
// It returns fs.ErrorIsDir error for directory objects, but still fills the
// fs.Object structure (for directory operations).
func (f *Fs) newObjectWithInfo(remote string, info *File) (fs.Object, error) {
if info == nil {
return nil, fs.ErrorObjectNotFound
}
URL := f.url(remote)
size := info.Size
if info.Type == "symlink" {
// File size for symlinks is absent but for .rclonelink to work
// the size should be the length of the target name
size = int64(len(info.Target))
}
o := &Object{
fs: f,
filetype: info.Type,
remote: remote,
size: size,
modTime: info.Mtime,
md5sum: info.Md5,
fullURL: URL,
target: info.Target,
}
if info.Type == "dir" {
return o, fs.ErrorIsDir
}
return o, nil
}
// getAuth is the signing hook to get the NetStorage auth
func (f *Fs) getAuth(req *http.Request) error {
// Set Authorization header
dataHeader := generateDataHeader(f)
path := req.URL.RequestURI()
//lint:ignore SA1008 false positive when running staticcheck, the header name is according to docs even if not canonical
//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1008
actionHeader := req.Header["X-Akamai-ACS-Action"][0]
fs.Debugf(nil, "NetStorage API %s call %s for path %q", req.Method, actionHeader, path)
req.Header.Set("X-Akamai-ACS-Auth-Data", dataHeader)
req.Header.Set("X-Akamai-ACS-Auth-Sign", generateSignHeader(f, dataHeader, path, actionHeader))
return nil
}
// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
423, // Locked
429, // Too Many Requests
500, // Internal Server Error
502, // Bad Gateway
503, // Service Unavailable
504, // Gateway Timeout
509, // Bandwidth Limit Exceeded
}
// shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}
// callBackend calls NetStorage API using either rest.Call or rest.CallXML function,
// depending on whether the response is required
func (f *Fs) callBackend(ctx context.Context, URL, method, actionHeader string, noResponse bool, response interface{}, options []fs.OpenOption) (io.ReadCloser, error) {
opts := rest.Opts{
Method: method,
RootURL: URL,
NoResponse: noResponse,
ExtraHeaders: map[string]string{
"*X-Akamai-ACS-Action": actionHeader,
},
}
if options != nil {
opts.Options = options
}
var resp *http.Response
err := f.pacer.Call(func() (bool, error) {
var err error
if response != nil {
resp, err = f.srv.CallXML(ctx, &opts, nil, response)
} else {
resp, err = f.srv.Call(ctx, &opts)
}
return shouldRetry(ctx, resp, err)
})
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
// 404 HTTP code translates into Object not found
return nil, fs.ErrorObjectNotFound
}
return nil, fmt.Errorf("failed to call NetStorage API: %w", err)
}
if noResponse {
return nil, nil
}
return resp.Body, nil
}
// netStorageStatRequest performs a NetStorage stat request
func (f *Fs) netStorageStatRequest(ctx context.Context, URL string, directory bool) ([]File, error) {
if strings.HasSuffix(URL, ".rclonelink") {
fs.Infof(nil, "Converting rclonelink to a symlink on the stat request %q", URL)
URL = strings.TrimSuffix(URL, ".rclonelink")
}
URL = strings.TrimSuffix(URL, "/")
files := f.getStatCache(URL)
if files == nil {
const actionHeader = "version=1&action=stat&implicit=yes&format=xml&encoding=utf-8&slash=both"
statResp := &Stat{}
if _, err := f.callBackend(ctx, URL, "GET", actionHeader, false, statResp, nil); err != nil {
fs.Debugf(nil, "NetStorage action stat failed for %q: %v", URL, err)
return nil, err
}
files = statResp.Files
f.setStatCache(URL, files)
}
// Multiple objects can be returned with the "slash=both" option,
// when file/symlink/directory has the same name
for i := range files {
if files[i].Type == "symlink" {
// Add .rclonelink suffix to allow local backend code to convert to a symlink.
files[i].Name += ".rclonelink"
fs.Infof(nil, "Converting a symlink to the rclonelink on the stat request %s", files[i].Name)
}
entrywanted := (directory && files[i].Type == "dir") ||
(!directory && files[i].Type != "dir")
if entrywanted {
filestamp := files[0]
files[0] = files[i]
files[i] = filestamp
}
}
return files, nil
}
// netStorageDirRequest performs a NetStorage dir request
func (f *Fs) netStorageDirRequest(ctx context.Context, dir string, URL string) ([]File, error) {
const actionHeader = "version=1&action=dir&format=xml&encoding=utf-8"
statResp := &Stat{}
if _, err := f.callBackend(ctx, URL, "GET", actionHeader, false, statResp, nil); err != nil {
if err == fs.ErrorObjectNotFound {
return nil, fs.ErrorDirNotFound
}
fs.Debugf(nil, "NetStorage action dir failed for %q: %v", URL, err)
return nil, err
}
return statResp.Files, nil
}
// netStorageListRequest performs a NetStorage list request
// Second returning parameter is resumeStart string, if not empty the function should be restarted with the adjusted URL to continue the listing.
func (f *Fs) netStorageListRequest(ctx context.Context, URL, endPath string) ([]File, string, error) {
actionHeader := "version=1&action=list&mtime_all=yes&format=xml&encoding=utf-8"
if !pathIsOneLevelDeep(endPath) {
// Add end= to limit the depth to endPath
escapeEndPath := url.QueryEscape(strings.TrimSuffix(endPath, "/"))
// The "0" character exists in place of the trailing slash to
// accommodate ObjectStore directory logic
end := "&end=" + strings.TrimSuffix(escapeEndPath, "/") + "0"
actionHeader += end
}
listResp := &List{}
if _, err := f.callBackend(ctx, URL, "GET", actionHeader, false, listResp, nil); err != nil {
if err == fs.ErrorObjectNotFound {
// List action is known to return 404 for a valid [CP Code] path with no objects inside.
// Call stat to find out whether it is an empty directory or path does not exist.
fs.Debugf(nil, "NetStorage action list returned 404, call stat for %q", URL)
files, err := f.netStorageStatRequest(ctx, URL, true)
if err == nil && len(files) > 0 && files[0].Type == "dir" {
return []File{}, "", nil
}
}
fs.Debugf(nil, "NetStorage action list failed for %q: %v", URL, err)
return nil, "", err
}
return listResp.Files, listResp.Resume.Start, nil
}
// netStorageUploadRequest performs a NetStorage upload request
func (o *Object) netStorageUploadRequest(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
URL := o.fullURL
if URL == "" {
URL = o.fs.url(src.Remote())
}
if strings.HasSuffix(URL, ".rclonelink") {
bits, err := io.ReadAll(in)
if err != nil {
return err
}
targ := string(bits)
symlinkloc := strings.TrimSuffix(URL, ".rclonelink")
fs.Infof(nil, "Converting rclonelink to a symlink on upload %s target %s", symlinkloc, targ)
_, err = o.fs.netStorageSymlinkRequest(ctx, symlinkloc, targ, &o.modTime)
return err
}
u, err := url.Parse(URL)
if err != nil {
return fmt.Errorf("unable to parse URL %q while uploading: %w", URL, err)
}
path := u.RequestURI()
const actionHeader = "version=1&action=upload&sha256=atend&mtime=atend"
trailers := &http.Header{}
hr := newHashReader(in, sha256.New())
reader := customReader(
func(p []byte) (n int, err error) {
if n, err = hr.Read(p); err != nil && err == io.EOF {
// Send the "chunked trailer" after upload of the object
digest := hex.EncodeToString(hr.Sum(nil))
actionHeader := "version=1&action=upload&sha256=" + digest +
"&mtime=" + strconv.FormatInt(src.ModTime(ctx).Unix(), 10)
trailers.Add("X-Akamai-ACS-Action", actionHeader)
dataHeader := generateDataHeader(o.fs)
trailers.Add("X-Akamai-ACS-Auth-Data", dataHeader)
signHeader := generateSignHeader(o.fs, dataHeader, path, actionHeader)
trailers.Add("X-Akamai-ACS-Auth-Sign", signHeader)
}
return
},
)
var resp *http.Response
opts := rest.Opts{
Method: "PUT",
RootURL: URL,
NoResponse: true,
Options: options,
Body: reader,
Trailer: trailers,
ExtraHeaders: map[string]string{
"*X-Akamai-ACS-Action": actionHeader,
},
}
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(ctx, resp, err)
})
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
// 404 HTTP code translates into Object not found
return fs.ErrorObjectNotFound
}
fs.Debugf(nil, "NetStorage action upload failed for %q: %v", URL, err)
// Remove failed upload
_ = o.Remove(ctx)
return fmt.Errorf("failed to call NetStorage API upload: %w", err)
}
// Invalidate stat cache
o.fs.deleteStatCache(URL)
if o.size == -1 {
files, err := o.fs.netStorageStatRequest(ctx, URL, false)
if err != nil {
return nil
}
if files != nil {
o.size = files[0].Size
}
}
return nil
}
// netStorageDownloadRequest performs a NetStorage download request
func (o *Object) netStorageDownloadRequest(ctx context.Context, options []fs.OpenOption) (in io.ReadCloser, err error) {
URL := o.fullURL
// If requested file ends with .rclonelink and target has value
// then serve the content of target (the symlink target)
if strings.HasSuffix(URL, ".rclonelink") && o.target != "" {
fs.Infof(nil, "Converting a symlink to the rclonelink file on download %q", URL)
reader := strings.NewReader(o.target)
readcloser := io.NopCloser(reader)
return readcloser, nil
}
const actionHeader = "version=1&action=download"
fs.FixRangeOption(options, o.size)
body, err := o.fs.callBackend(ctx, URL, "GET", actionHeader, false, nil, options)
if err != nil {
fs.Debugf(nil, "NetStorage action download failed for %q: %v", URL, err)
return nil, err
}
return body, nil
}
// netStorageDuRequest performs a NetStorage du request
func (f *Fs) netStorageDuRequest(ctx context.Context) (interface{}, error) {
URL := f.url("")
const actionHeader = "version=1&action=du&format=xml&encoding=utf-8"
duResp := &Du{}
if _, err := f.callBackend(ctx, URL, "GET", actionHeader, false, duResp, nil); err != nil {
if err == fs.ErrorObjectNotFound {
return nil, errors.New("NetStorage du command: target is not a directory or does not exist")
}
fs.Debugf(nil, "NetStorage action du failed for %q: %v", URL, err)
return nil, err
}
//passing the output format expected from return of Command to be displayed by rclone code
out := map[string]int64{
"Number of files": duResp.Duinfo.Files,
"Total bytes": duResp.Duinfo.Bytes,
}
return out, nil
}
// netStorageDuRequest performs a NetStorage symlink request
func (f *Fs) netStorageSymlinkRequest(ctx context.Context, URL string, dst string, modTime *int64) (interface{}, error) {
target := url.QueryEscape(strings.TrimSuffix(dst, "/"))
actionHeader := "version=1&action=symlink&target=" + target
if modTime != nil {
when := strconv.FormatInt(*modTime, 10)
actionHeader += "&mtime=" + when
}
if _, err := f.callBackend(ctx, URL, "POST", actionHeader, true, nil, nil); err != nil {
fs.Debugf(nil, "NetStorage action symlink failed for %q: %v", URL, err)
return nil, fmt.Errorf("symlink creation failed: %w", err)
}
f.deleteStatCache(URL)
out := map[string]string{
"Symlink successfully created": dst,
}
return out, nil
}
// netStorageMkdirRequest performs a NetStorage mkdir request
func (f *Fs) netStorageMkdirRequest(ctx context.Context, URL string) error {
const actionHeader = "version=1&action=mkdir"
if _, err := f.callBackend(ctx, URL, "POST", actionHeader, true, nil, nil); err != nil {
fs.Debugf(nil, "NetStorage action mkdir failed for %q: %v", URL, err)
return err
}
f.deleteStatCache(URL)
return nil
}
// netStorageDeleteRequest performs a NetStorage delete request
func (o *Object) netStorageDeleteRequest(ctx context.Context) error {
URL := o.fullURL
// We shouldn't be creating .rclonelink files on remote
// but delete corresponding symlink if it exists
if strings.HasSuffix(URL, ".rclonelink") {
fs.Infof(nil, "Converting rclonelink to a symlink on delete %q", URL)
URL = strings.TrimSuffix(URL, ".rclonelink")
}
const actionHeader = "version=1&action=delete"
if _, err := o.fs.callBackend(ctx, URL, "POST", actionHeader, true, nil, nil); err != nil {
fs.Debugf(nil, "NetStorage action delete failed for %q: %v", URL, err)
return err
}
o.fs.deleteStatCache(URL)
return nil
}
// netStorageRmdirRequest performs a NetStorage rmdir request
func (f *Fs) netStorageRmdirRequest(ctx context.Context, dir string) error {
URL := f.url(dir)
const actionHeader = "version=1&action=rmdir"
if _, err := f.callBackend(ctx, URL, "POST", actionHeader, true, nil, nil); err != nil {
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
}
fs.Debugf(nil, "NetStorage action rmdir failed for %q: %v", URL, err)
return err
}
f.deleteStatCache(URL)
f.deleteDirscreated(URL)
return nil
}
// deleteDirscreated deletes URL from dirscreated map thread-safely
func (f *Fs) deleteDirscreated(URL string) {
URL = strings.TrimSuffix(URL, "/")
f.dirscreatedMutex.Lock()
delete(f.dirscreated, URL)
f.dirscreatedMutex.Unlock()
}
// setDirscreated sets to true URL in dirscreated map thread-safely
func (f *Fs) setDirscreated(URL string) {
URL = strings.TrimSuffix(URL, "/")
f.dirscreatedMutex.Lock()
f.dirscreated[URL] = true
f.dirscreatedMutex.Unlock()
}
// testAndSetDirscreated atomic test-and-set to true URL in dirscreated map,
// returns the previous value
func (f *Fs) testAndSetDirscreated(URL string) bool {
URL = strings.TrimSuffix(URL, "/")
f.dirscreatedMutex.Lock()
oldValue := f.dirscreated[URL]
f.dirscreated[URL] = true
f.dirscreatedMutex.Unlock()
return oldValue
}
// deleteStatCache deletes URL from stat cache thread-safely
func (f *Fs) deleteStatCache(URL string) {
URL = strings.TrimSuffix(URL, "/")
f.statcacheMutex.Lock()
delete(f.statcache, URL)
f.statcacheMutex.Unlock()
}
// getStatCache gets value from statcache map thread-safely
func (f *Fs) getStatCache(URL string) (files []File) {
URL = strings.TrimSuffix(URL, "/")
f.statcacheMutex.RLock()
files = f.statcache[URL]
f.statcacheMutex.RUnlock()
if files != nil {
fs.Debugf(nil, "NetStorage stat cache hit for %q", URL)
}
return
}
// setStatCache sets value to statcache map thread-safely
func (f *Fs) setStatCache(URL string, files []File) {
URL = strings.TrimSuffix(URL, "/")
f.statcacheMutex.Lock()
f.statcache[URL] = files
f.statcacheMutex.Unlock()
}
type hashReader struct {
io.Reader
gohash.Hash
}
func newHashReader(r io.Reader, h gohash.Hash) hashReader {
return hashReader{io.TeeReader(r, h), h}
}
type customReader func([]byte) (int, error)
func (c customReader) Read(p []byte) (n int, err error) {
return c(p)
}
// generateRequestID generates the unique requestID
func generateRequestID() int64 {
s1 := rand.NewSource(time.Now().UnixNano())
r1 := rand.New(s1)
return r1.Int63()
}
// computeHmac256 calculates the hash for the sign header
func computeHmac256(message string, secret string) string {
key := []byte(secret)
h := hmac.New(sha256.New, key)
h.Write([]byte(message))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
// getEpochTimeInSeconds returns current epoch time in seconds
func getEpochTimeInSeconds() int64 {
now := time.Now()
secs := now.Unix()
return secs
}
// generateDataHeader generates data header needed for making the request
func generateDataHeader(f *Fs) string {
return "5, 0.0.0.0, 0.0.0.0, " + strconv.FormatInt(getEpochTimeInSeconds(), 10) + ", " + strconv.FormatInt(generateRequestID(), 10) + "," + f.opt.Account
}
// generateSignHeader generates sign header needed for making the request
func generateSignHeader(f *Fs, dataHeader string, path string, actionHeader string) string {
var message = dataHeader + path + "\nx-akamai-acs-action:" + actionHeader + "\n"
return computeHmac256(message, f.opt.Secret)
}
// pathIsOneLevelDeep returns true if a given path does not go deeper than one level
func pathIsOneLevelDeep(path string) bool {
return !strings.Contains(strings.TrimSuffix(strings.TrimPrefix(path, "/"), "/"), "/")
}
// Check the interfaces are satisfied
var (
_ fs.Fs = &Fs{}
_ fs.Purger = &Fs{}
_ fs.PutStreamer = &Fs{}
_ fs.ListRer = &Fs{}
_ fs.Object = &Object{}
)