From fcea3777c0103c33e2dd5bc59a6f9c9a0a4819e0 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 8 Nov 2015 15:29:58 +0000 Subject: [PATCH] Implement Hubic storage system - fixes #200 --- docs/content/about.md | 1 + docs/content/hubic.md | 98 ++++++++++++++ docs/content/overview.md | 1 + docs/layouts/chrome/navbar.html | 1 + fs/operations_test.go | 1 + fs/test_all.sh | 1 + fstest/fstests/fstests.go | 17 ++- fstest/fstests/gen_tests.go | 1 + hubic/auth.go | 54 ++++++++ hubic/hubic.go | 226 ++++++++++++++++++++++++++++++++ hubic/hubic_test.go | 56 ++++++++ make_manual.py | 1 + rclone.go | 1 + swift/swift.go | 18 ++- 14 files changed, 469 insertions(+), 8 deletions(-) create mode 100644 docs/content/hubic.md create mode 100644 hubic/auth.go create mode 100644 hubic/hubic.go create mode 100644 hubic/hubic_test.go diff --git a/docs/content/about.md b/docs/content/about.md index b45a2c4d9..f0b93a1e8 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -20,6 +20,7 @@ Rclone is a command line program to sync files and directories to and from * Google Cloud Storage * Amazon Cloud Drive * Microsoft One Drive + * Hubic * The local filesystem Features diff --git a/docs/content/hubic.md b/docs/content/hubic.md new file mode 100644 index 000000000..3595069ef --- /dev/null +++ b/docs/content/hubic.md @@ -0,0 +1,98 @@ +--- +title: "Hubic" +description: "Rclone docs for Hubic" +date: "2015-11-08" +--- + + Hubic +----------------------------------------- + +Paths are specified as `remote:path` + +Paths are specified as `remote:container` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg `remote:container/path/to/dir`. + +The initial setup for Hubic involves getting a token from Hubic which +you need to do in your browser. `rclone config` walks you through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +n) New remote +d) Delete remote +q) Quit config +e/n/d/q> n +name> remote +What type of source is it? +Choose a number from below + 1) amazon cloud drive + 2) drive + 3) dropbox + 4) google cloud storage + 5) local + 6) onedrive + 7) hubic + 8) s3 + 9) swift +type> 7 +Hubic App Client Id - leave blank normally. +client_id> +Hubic App Client Secret - leave blank normally. +client_secret> +Remote config +If your browser doesn't open automatically go to the following link: http://localhost:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +client_id = +client_secret = +token = {"access_token":"XXXXXX"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Hubic. This only runs from the moment it opens +your browser to the moment you get back the verification code. This +is on `http://127.0.0.1:53682/` and this it may require you to unblock +it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +List containers in the top level of your Hubic + + rclone lsd remote: + +List all the files in your Hubic + + rclone ls remote: + +To copy a local directory to an Hubic directory called backup + + rclone copy /home/source remote:backup + +### Modified time ### + +The modified time is stored as metadata on the object as +`X-Object-Meta-Mtime` as floating point since the epoch accurate to 1 +ns. + +This is a defacto standard (used in the official python-swiftclient +amongst others) for storing the modification time for an object. + +Note that Hubic wraps the Swift backend, so most of the properties of +are the same. + +### Limitations ### + +Code to refresh the OpenStack token isn't done yet which may cause +problems with very long transfers. diff --git a/docs/content/overview.md b/docs/content/overview.md index 56db86982..b3a2f7e2a 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -24,6 +24,7 @@ Here is an overview of the major features of each cloud storage system. | Google Cloud Storage | Yes | Yes | No | No | | Amazon Cloud Drive | Yes | No | Yes | No | | Microsoft One Drive | No | Yes | Yes | No | +| Hubic | Yes | Yes | No | No | | The local filesystem | Yes | Yes | Depends | No | ### MD5SUM ### diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index d0ae7f562..0fb445690 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -38,6 +38,7 @@
  • Google Cloud Storage
  • Amazon Cloud Drive
  • Microsoft One Drive
    • +
  • Hubic
  • Local
    • diff --git a/fs/operations_test.go b/fs/operations_test.go index fcebed04c..625a67a3f 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -24,6 +24,7 @@ import ( _ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/googlecloudstorage" + _ "github.com/ncw/rclone/hubic" _ "github.com/ncw/rclone/local" _ "github.com/ncw/rclone/onedrive" _ "github.com/ncw/rclone/s3" diff --git a/fs/test_all.sh b/fs/test_all.sh index 7f2296823..b9ad722d3 100755 --- a/fs/test_all.sh +++ b/fs/test_all.sh @@ -10,6 +10,7 @@ TestGoogleCloudStorage: TestDropbox: TestAmazonCloudDrive: TestOneDrive: +TestHubic: " function test_remote { diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 87bb6a008..3e5280ef2 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -435,7 +435,14 @@ func TestObjectString(t *testing.T) { func TestObjectFs(t *testing.T) { skipIfNotOk(t) obj := findObject(t, file1.Path) - if obj.Fs() != remote { + equal := obj.Fs() == remote + if !equal { + // Check to see if this wraps something else + if unwrap, ok := remote.(fs.UnWrapper); ok { + equal = obj.Fs() == unwrap.UnWrap() + } + } + if !equal { t.Errorf("Fs is wrong %v != %v", obj.Fs(), remote) } } @@ -558,7 +565,13 @@ func TestLimitedFs(t *testing.T) { fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy}) _, ok := fileRemote.(*fs.Limited) if !ok { - t.Errorf("%v is not a fs.Limited", fileRemote) + // Check to see if this wraps a Limited FS + if unwrap, hasUnWrap := fileRemote.(fs.UnWrapper); hasUnWrap { + _, ok = unwrap.UnWrap().(*fs.Limited) + } + if !ok { + t.Errorf("%v is not a fs.Limited", fileRemote) + } } } diff --git a/fstest/fstests/gen_tests.go b/fstest/fstests/gen_tests.go index a58070ae7..52b8871aa 100644 --- a/fstest/fstests/gen_tests.go +++ b/fstest/fstests/gen_tests.go @@ -132,5 +132,6 @@ func main() { generateTestProgram(t, fns, "Dropbox") generateTestProgram(t, fns, "AmazonCloudDrive") generateTestProgram(t, fns, "OneDrive") + generateTestProgram(t, fns, "Hubic") log.Printf("Done") } diff --git a/hubic/auth.go b/hubic/auth.go new file mode 100644 index 000000000..c3196cdce --- /dev/null +++ b/hubic/auth.go @@ -0,0 +1,54 @@ +package hubic + +import ( + "net/http" + + "github.com/ncw/swift" +) + +// auth is an authenticator for swift +type auth struct { + f *Fs +} + +// newAuth creates a swift authenticator +func newAuth(f *Fs) *auth { + return &auth{ + f: f, + } +} + +// Request constructs a http.Request for authentication +// +// returns nil for not needed +func (a *auth) Request(*swift.Connection) (*http.Request, error) { + err := a.f.getCredentials() + if err != nil { + return nil, err + } + return nil, nil +} + +// Response parses the result of an http request +func (a *auth) Response(resp *http.Response) error { + return nil +} + +// The public storage URL - set Internal to true to read +// internal/service net URL +func (a *auth) StorageUrl(Internal bool) string { + return a.f.credentials.Endpoint +} + +// The access token +func (a *auth) Token() string { + return a.f.credentials.Token +} + +// The CDN url if available +func (a *auth) CdnUrl() string { + return "" +} + +// Check the interfaces are satisfied +var _ swift.Authenticator = (*auth)(nil) diff --git a/hubic/hubic.go b/hubic/hubic.go new file mode 100644 index 000000000..35f3047e7 --- /dev/null +++ b/hubic/hubic.go @@ -0,0 +1,226 @@ +// Package hubic provides an interface to the Hubic object storage +// system. +package hubic + +// This uses the normal swift mechanism to update the credentials and +// ignores the expires field returned by the Hubic API. This may need +// to be revisted after some actual experience. + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/oauthutil" + "github.com/ncw/rclone/swift" + swiftLib "github.com/ncw/swift" + "golang.org/x/oauth2" +) + +const ( + rcloneClientID = "api_hubic_svWP970PvSWbw5G3PzrAqZ6X2uHeZBPI" + rcloneClientSecret = "8MrG3pjWyJya4OnO9ZTS4emI+9fa1ouPgvfD2MbTzfDYvO/H5czFxsTXtcji4/Hz3snz8/CrzMzlxvP9//Ty/Q==" +) + +// Globals +var ( + // Description of how to auth for this app + oauthConfig = &oauth2.Config{ + Scopes: []string{ + "credentials.r", // Read Openstack credentials + }, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://api.hubic.com/oauth/auth/", + TokenURL: "https://api.hubic.com/oauth/token/", + }, + ClientID: rcloneClientID, + ClientSecret: fs.Reveal(rcloneClientSecret), + RedirectURL: oauthutil.RedirectLocalhostURL, + } +) + +// Register with Fs +func init() { + fs.Register(&fs.Info{ + Name: "hubic", + NewFs: NewFs, + Config: func(name string) { + err := oauthutil.Config(name, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: []fs.Option{{ + Name: oauthutil.ConfigClientID, + Help: "Hubic Client Id - leave blank normally.", + }, { + Name: oauthutil.ConfigClientSecret, + Help: "Hubic Client Secret - leave blank normally.", + }}, + }) +} + +// credentials is the JSON returned from the Hubic API to read the +// OpenStack credentials +type credentials struct { + Token string `json:"token"` // Openstack token + Endpoint string `json:"endpoint"` // Openstack endpoint + Expires string `json:"expires"` // Expires date - eg "2015-11-09T14:24:56+01:00" +} + +// Fs represents a remote hubic +type Fs struct { + fs.Fs // wrapped Fs + client *http.Client // client for oauth api + credentials credentials // returned from the Hubic API + expires time.Time // time credentials expire +} + +// Object describes a swift object +type Object struct { + *swift.Object +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.Object.String() +} + +// ------------------------------------------------------------ + +// String converts this Fs to a string +func (f *Fs) String() string { + if f.Fs == nil { + return "Hubic" + } + return fmt.Sprintf("Hubic %s", f.Fs.String()) +} + +// checkClose is a utility function used to check the return from +// Close in a defer statement. +func checkClose(c io.Closer, err *error) { + cerr := c.Close() + if *err == nil { + *err = cerr + } +} + +// getCredentials reads the OpenStack Credentials using the Hubic API +// +// The credentials are read into the Fs +func (f *Fs) getCredentials() (err error) { + req, err := http.NewRequest("GET", "https://api.hubic.com/1.0/account/credentials", nil) + if err != nil { + return err + } + req.Header.Add("User-Agent", fs.UserAgent) + resp, err := f.client.Do(req) + if err != nil { + return err + } + defer checkClose(resp.Body, &err) + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return fmt.Errorf("Failed to get credentials: %s", resp.Status) + } + decoder := json.NewDecoder(resp.Body) + var result credentials + err = decoder.Decode(&result) + if err != nil { + return err + } + // fs.Debug(f, "Got credentials %+v", result) + if result.Token == "" || result.Endpoint == "" || result.Expires == "" { + return fmt.Errorf("Couldn't read token, result and expired from credentials") + } + f.credentials = result + expires, err := time.Parse(time.RFC3339, result.Expires) + if err != nil { + return err + } + f.expires = expires + fs.Debug(f, "Got swift credentials (expiry %v in %v)", f.expires, f.expires.Sub(time.Now())) + return nil +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string) (fs.Fs, error) { + client, err := oauthutil.NewClient(name, oauthConfig) + if err != nil { + return nil, fmt.Errorf("Failed to configure Hubic: %v", err) + } + + f := &Fs{ + client: client, + } + + // Make the swift Connection + c := &swiftLib.Connection{ + Auth: newAuth(f), + UserAgent: fs.UserAgent, + ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport + Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport + Transport: fs.Config.Transport(), + } + err = c.Authenticate() + if err != nil { + return nil, fmt.Errorf("Error authenticating swift connection: %v", err) + } + + // Make inner swift Fs from the connection + swiftFs, err := swift.NewFsWithConnection(name, root, c) + if err != nil { + return nil, err + } + f.Fs = swiftFs + return f, nil +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + fPurge, ok := f.Fs.(fs.Purger) + if !ok { + return fs.ErrorCantPurge + } + return fPurge.Purge() +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + fCopy, ok := f.Fs.(fs.Copier) + if !ok { + return nil, fs.ErrorCantCopy + } + return fCopy.Copy(src, remote) +} + +// UnWrap returns the Fs that this Fs is wrapping +func (f *Fs) UnWrap() fs.Fs { + return f.Fs +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.UnWrapper = (*Fs)(nil) +) diff --git a/hubic/hubic_test.go b/hubic/hubic_test.go new file mode 100644 index 000000000..84e344dc4 --- /dev/null +++ b/hubic/hubic_test.go @@ -0,0 +1,56 @@ +// Test Hubic filesystem interface +// +// Automatically generated - DO NOT EDIT +// Regenerate with: make gen_tests +package hubic_test + +import ( + "testing" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" + "github.com/ncw/rclone/hubic" +) + +func init() { + fstests.NilObject = fs.Object((*hubic.Object)(nil)) + fstests.RemoteName = "TestHubic:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/make_manual.py b/make_manual.py index 7cb9ed927..e14309b69 100755 --- a/make_manual.py +++ b/make_manual.py @@ -25,6 +25,7 @@ docs = [ "googlecloudstorage.md", "amazonclouddrive.md", "onedrive.md", + "hubic.md", "local.md", "changelog.md", "bugs.md", diff --git a/rclone.go b/rclone.go index ae6784020..878ddfb50 100644 --- a/rclone.go +++ b/rclone.go @@ -20,6 +20,7 @@ import ( _ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/googlecloudstorage" + _ "github.com/ncw/rclone/hubic" _ "github.com/ncw/rclone/local" _ "github.com/ncw/rclone/onedrive" _ "github.com/ncw/rclone/s3" diff --git a/swift/swift.go b/swift/swift.go index 63ad5ba7b..6e98c03ff 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -159,16 +159,13 @@ func swiftConnection(name string) (*swift.Connection, error) { return c, nil } -// NewFs contstructs an Fs from the path, container:path -func NewFs(name, root string) (fs.Fs, error) { +// NewFsWithConnection contstructs an Fs from the path, container:path +// and authenticated connection +func NewFsWithConnection(name, root string, c *swift.Connection) (fs.Fs, error) { container, directory, err := parsePath(root) if err != nil { return nil, err } - c, err := swiftConnection(name) - if err != nil { - return nil, err - } f := &Fs{ name: name, c: *c, @@ -196,6 +193,15 @@ func NewFs(name, root string) (fs.Fs, error) { return f, nil } +// NewFs contstructs an Fs from the path, container:path +func NewFs(name, root string) (fs.Fs, error) { + c, err := swiftConnection(name) + if err != nil { + return nil, err + } + return NewFsWithConnection(name, root, c) +} + // Return an FsObject from a path // // May return nil if an error occurred