rclone/swift/fs.go

// Swift interface
package swift

// FIXME need to prevent anything but ListDir working for swift://

import (
	"errors"
	"flag"
	"fmt"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/swift"
	"io"
	"log"
	"os"
	"regexp"
	"strings"
	"time"
)

// Pattern to match a swift url
var Match = regexp.MustCompile(`^swift://([^/]*)(.*)$`)

// Register with Fs
func init() {
	fs.Register(Match, NewFs)
}

// FsSwift represents a remote swift server
type FsSwift struct {
	c         swift.Connection // the connection to the swift server
	container string           // the container we are working on
	root      string           // the path we are working on if any
}

// FsObjectSwift describes a swift object
//
// Will definitely have info but maybe not meta
type FsObjectSwift struct {
	swift  *FsSwift        // what this object is part of
	remote string          // The remote path
	info   swift.Object    // Info from the swift object if known
	meta   *swift.Metadata // The object metadata if known
}

// ------------------------------------------------------------

// Globals
var (
	// Flags
	// FIXME make these part of swift so we get a standard set of flags?
	authUrl  = flag.String("swift-auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
	userName = flag.String("swift-user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
	apiKey   = flag.String("swift-key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
	snet     = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
)

// String converts this FsSwift to a string
func (f *FsSwift) String() string {
	return fmt.Sprintf("Swift container %s", f.container)
}

// parseParse parses a swift 'url'
func parsePath(path string) (container, directory string, err error) {
	parts := Match.FindAllStringSubmatch(path, -1)
	if len(parts) != 1 || len(parts[0]) != 3 {
		err = fmt.Errorf("Couldn't parse swift url %q", path)
	} else {
		container, directory = parts[0][1], parts[0][2]
		directory = strings.Trim(directory, "/")
	}
	return
}

// swiftConnection makes a connection to swift
func swiftConnection() (*swift.Connection, error) {
	if *userName == "" {
		return nil, errors.New("Need -user or environmental variable ST_USER")
	}
	if *apiKey == "" {
		return nil, errors.New("Need -key or environmental variable ST_KEY")
	}
	if *authUrl == "" {
		return nil, errors.New("Need -auth or environmental variable ST_AUTH")
	}
	c := &swift.Connection{
		UserName: *userName,
		ApiKey:   *apiKey,
		AuthUrl:  *authUrl,
	}
	err := c.Authenticate()
	if err != nil {
		return nil, err
	}
	return c, nil
}

// NewFs contstructs an FsSwift from the path, container:path
func NewFs(path string) (fs.Fs, error) {
	container, directory, err := parsePath(path)
	if err != nil {
		return nil, err
	}
	if directory != "" {
		return nil, fmt.Errorf("Directories not supported yet in %q", path)
	}
	c, err := swiftConnection()
	if err != nil {
		return nil, err
	}
	f := &FsSwift{c: *c, container: container, root: directory}
	return f, nil
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
	fs := &FsObjectSwift{
		swift:  f,
		remote: remote,
	}
	if info != nil {
		// Set info but not meta
		fs.info = *info
	} else {
		err := fs.readMetaData() // reads info and meta, returning an error
		if err != nil {
			// logged already FsDebug("Failed to read info: %s", err)
			return nil
		}
	}
	return fs
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsSwift) NewFsObject(remote string) fs.Object {
	return f.NewFsObjectWithInfo(remote, nil)
}

// Walk the path returning a channel of FsObjects
func (f *FsSwift) List() fs.ObjectsChan {
	out := make(fs.ObjectsChan, fs.Config.Checkers)
	go func() {
		// FIXME use a smaller limit?
		err := f.c.ObjectsWalk(f.container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) {
			objects, err := f.c.Objects(f.container, opts)
			if err == nil {
				for i := range objects {
					object := &objects[i]
					if fs := f.NewFsObjectWithInfo(object.Name, object); fs != nil {
						out <- fs
					}
				}
			}
			return objects, err
		})
		if err != nil {
			fs.Stats.Error()
			log.Printf("Couldn't read container %q: %s", f.container, err)
		}
		close(out)
	}()
	return out
}

// Lists the containers
func (f *FsSwift) ListDir() fs.DirChan {
	out := make(fs.DirChan, fs.Config.Checkers)
	go func() {
		defer close(out)
		containers, err := f.c.ContainersAll(nil)
		if err != nil {
			fs.Stats.Error()
			log.Printf("Couldn't list containers: %s", err)
		} else {
			for _, container := range containers {
				out <- &fs.Dir{
					Name:  container.Name,
					Bytes: container.Bytes,
					Count: container.Count,
				}
			}
		}
	}()
	return out
}

// Put the FsObject into the container
//
// Copy the reader in to the new object which is returned
//
// The new object may have been created
func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
	// Temporary FsObject under construction
	fs := &FsObjectSwift{swift: f, remote: remote}

	// Set the mtime
	m := swift.Metadata{}
	m.SetModTime(modTime)
	_, err := f.c.ObjectPut(f.container, remote, in, true, "", "", m.ObjectHeaders())
	return fs, err
}

// Mkdir creates the container if it doesn't exist
func (f *FsSwift) Mkdir() error {
	return f.c.ContainerCreate(f.container, nil)
}

// Rmdir deletes the container
//
// Returns an error if it isn't empty
func (f *FsSwift) Rmdir() error {
	return f.c.ContainerDelete(f.container)
}

// Return the precision
func (fs *FsSwift) Precision() time.Duration {
	return time.Nanosecond
}

// ------------------------------------------------------------

// Return the remote path
func (o *FsObjectSwift) Remote() string {
	return o.remote
}

// Md5sum returns the Md5sum of an object returning a lowercase hex string
func (o *FsObjectSwift) Md5sum() (string, error) {
	return strings.ToLower(o.info.Hash), nil
}

// Size returns the size of an object in bytes
func (o *FsObjectSwift) Size() int64 {
	return o.info.Bytes
}

// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
func (o *FsObjectSwift) readMetaData() (err error) {
	if o.meta != nil {
		return nil
	}
	info, h, err := o.swift.c.Object(o.swift.container, o.remote)
	if err != nil {
		fs.Debug(o, "Failed to read info: %s", err)
		return err
	}
	meta := h.ObjectMetadata()
	o.info = info
	o.meta = &meta
	return nil
}

// ModTime returns the modification time of the object
//
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *FsObjectSwift) ModTime() time.Time {
	err := o.readMetaData()
	if err != nil {
		// fs.Log(o, "Failed to read metadata: %s", err)
		return o.info.LastModified
	}
	modTime, err := o.meta.GetModTime()
	if err != nil {
		// fs.Log(o, "Failed to read mtime from object: %s", err)
		return o.info.LastModified
	}
	return modTime
}

// Sets the modification time of the local fs object
func (o *FsObjectSwift) SetModTime(modTime time.Time) {
	err := o.readMetaData()
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to read metadata: %s", err)
		return
	}
	o.meta.SetModTime(modTime)
	err = o.swift.c.ObjectUpdate(o.swift.container, o.remote, o.meta.ObjectHeaders())
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to update remote mtime: %s", err)
	}
}

// Is this object storable
func (o *FsObjectSwift) Storable() bool {
	return true
}

// Open an object for read
func (o *FsObjectSwift) Open() (in io.ReadCloser, err error) {
	in, _, err = o.swift.c.ObjectOpen(o.swift.container, o.remote, true, nil)
	return
}

// Remove an object
func (o *FsObjectSwift) Remove() error {
	return o.swift.c.ObjectDelete(o.swift.container, o.remote)
}

// Check the interfaces are satisfied
var _ fs.Fs = &FsSwift{}
var _ fs.Object = &FsObjectSwift{}