diff --git a/docs/content/about.md b/docs/content/about.md
index b45a2c4d9..f0b93a1e8 100644
--- a/docs/content/about.md
+++ b/docs/content/about.md
@@ -20,6 +20,7 @@ Rclone is a command line program to sync files and directories to and from
* Google Cloud Storage
* Amazon Cloud Drive
* Microsoft One Drive
+ * Hubic
* The local filesystem
Features
diff --git a/docs/content/hubic.md b/docs/content/hubic.md
new file mode 100644
index 000000000..3595069ef
--- /dev/null
+++ b/docs/content/hubic.md
@@ -0,0 +1,98 @@
+---
+title: "Hubic"
+description: "Rclone docs for Hubic"
+date: "2015-11-08"
+---
+
+ Hubic
+-----------------------------------------
+
+Paths are specified as `remote:path`
+
+Paths are specified as `remote:container` (or `remote:` for the `lsd`
+command.) You may put subdirectories in too, eg `remote:container/path/to/dir`.
+
+The initial setup for Hubic involves getting a token from Hubic which
+you need to do in your browser. `rclone config` walks you through it.
+
+Here is an example of how to make a remote called `remote`. First run:
+
+ rclone config
+
+This will guide you through an interactive setup process:
+
+```
+n) New remote
+d) Delete remote
+q) Quit config
+e/n/d/q> n
+name> remote
+What type of source is it?
+Choose a number from below
+ 1) amazon cloud drive
+ 2) drive
+ 3) dropbox
+ 4) google cloud storage
+ 5) local
+ 6) onedrive
+ 7) hubic
+ 8) s3
+ 9) swift
+type> 7
+Hubic App Client Id - leave blank normally.
+client_id>
+Hubic App Client Secret - leave blank normally.
+client_secret>
+Remote config
+If your browser doesn't open automatically go to the following link: http://localhost:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+client_id =
+client_secret =
+token = {"access_token":"XXXXXX"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Note that rclone runs a webserver on your local machine to collect the
+token as returned from Hubic. This only runs from the moment it opens
+your browser to the moment you get back the verification code. This
+is on `http://127.0.0.1:53682/` and this it may require you to unblock
+it temporarily if you are running a host firewall.
+
+Once configured you can then use `rclone` like this,
+
+List containers in the top level of your Hubic
+
+ rclone lsd remote:
+
+List all the files in your Hubic
+
+ rclone ls remote:
+
+To copy a local directory to an Hubic directory called backup
+
+ rclone copy /home/source remote:backup
+
+### Modified time ###
+
+The modified time is stored as metadata on the object as
+`X-Object-Meta-Mtime` as floating point since the epoch accurate to 1
+ns.
+
+This is a defacto standard (used in the official python-swiftclient
+amongst others) for storing the modification time for an object.
+
+Note that Hubic wraps the Swift backend, so most of the properties of
+are the same.
+
+### Limitations ###
+
+Code to refresh the OpenStack token isn't done yet which may cause
+problems with very long transfers.
diff --git a/docs/content/overview.md b/docs/content/overview.md
index 56db86982..b3a2f7e2a 100644
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@@ -24,6 +24,7 @@ Here is an overview of the major features of each cloud storage system.
| Google Cloud Storage | Yes | Yes | No | No |
| Amazon Cloud Drive | Yes | No | Yes | No |
| Microsoft One Drive | No | Yes | Yes | No |
+| Hubic | Yes | Yes | No | No |
| The local filesystem | Yes | Yes | Depends | No |
### MD5SUM ###
diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html
index d0ae7f562..0fb445690 100644
--- a/docs/layouts/chrome/navbar.html
+++ b/docs/layouts/chrome/navbar.html
@@ -38,6 +38,7 @@
Google Cloud Storage
Amazon Cloud Drive
Microsoft One Drive
+ Hubic
Local
diff --git a/fs/operations_test.go b/fs/operations_test.go
index fcebed04c..625a67a3f 100644
--- a/fs/operations_test.go
+++ b/fs/operations_test.go
@@ -24,6 +24,7 @@ import (
_ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/googlecloudstorage"
+ _ "github.com/ncw/rclone/hubic"
_ "github.com/ncw/rclone/local"
_ "github.com/ncw/rclone/onedrive"
_ "github.com/ncw/rclone/s3"
diff --git a/fs/test_all.sh b/fs/test_all.sh
index 7f2296823..b9ad722d3 100755
--- a/fs/test_all.sh
+++ b/fs/test_all.sh
@@ -10,6 +10,7 @@ TestGoogleCloudStorage:
TestDropbox:
TestAmazonCloudDrive:
TestOneDrive:
+TestHubic:
"
function test_remote {
diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go
index 87bb6a008..3e5280ef2 100644
--- a/fstest/fstests/fstests.go
+++ b/fstest/fstests/fstests.go
@@ -435,7 +435,14 @@ func TestObjectString(t *testing.T) {
func TestObjectFs(t *testing.T) {
skipIfNotOk(t)
obj := findObject(t, file1.Path)
- if obj.Fs() != remote {
+ equal := obj.Fs() == remote
+ if !equal {
+ // Check to see if this wraps something else
+ if unwrap, ok := remote.(fs.UnWrapper); ok {
+ equal = obj.Fs() == unwrap.UnWrap()
+ }
+ }
+ if !equal {
t.Errorf("Fs is wrong %v != %v", obj.Fs(), remote)
}
}
@@ -558,7 +565,13 @@ func TestLimitedFs(t *testing.T) {
fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
_, ok := fileRemote.(*fs.Limited)
if !ok {
- t.Errorf("%v is not a fs.Limited", fileRemote)
+ // Check to see if this wraps a Limited FS
+ if unwrap, hasUnWrap := fileRemote.(fs.UnWrapper); hasUnWrap {
+ _, ok = unwrap.UnWrap().(*fs.Limited)
+ }
+ if !ok {
+ t.Errorf("%v is not a fs.Limited", fileRemote)
+ }
}
}
diff --git a/fstest/fstests/gen_tests.go b/fstest/fstests/gen_tests.go
index a58070ae7..52b8871aa 100644
--- a/fstest/fstests/gen_tests.go
+++ b/fstest/fstests/gen_tests.go
@@ -132,5 +132,6 @@ func main() {
generateTestProgram(t, fns, "Dropbox")
generateTestProgram(t, fns, "AmazonCloudDrive")
generateTestProgram(t, fns, "OneDrive")
+ generateTestProgram(t, fns, "Hubic")
log.Printf("Done")
}
diff --git a/hubic/auth.go b/hubic/auth.go
new file mode 100644
index 000000000..c3196cdce
--- /dev/null
+++ b/hubic/auth.go
@@ -0,0 +1,54 @@
+package hubic
+
+import (
+ "net/http"
+
+ "github.com/ncw/swift"
+)
+
+// auth is an authenticator for swift
+type auth struct {
+ f *Fs
+}
+
+// newAuth creates a swift authenticator
+func newAuth(f *Fs) *auth {
+ return &auth{
+ f: f,
+ }
+}
+
+// Request constructs a http.Request for authentication
+//
+// returns nil for not needed
+func (a *auth) Request(*swift.Connection) (*http.Request, error) {
+ err := a.f.getCredentials()
+ if err != nil {
+ return nil, err
+ }
+ return nil, nil
+}
+
+// Response parses the result of an http request
+func (a *auth) Response(resp *http.Response) error {
+ return nil
+}
+
+// The public storage URL - set Internal to true to read
+// internal/service net URL
+func (a *auth) StorageUrl(Internal bool) string {
+ return a.f.credentials.Endpoint
+}
+
+// The access token
+func (a *auth) Token() string {
+ return a.f.credentials.Token
+}
+
+// The CDN url if available
+func (a *auth) CdnUrl() string {
+ return ""
+}
+
+// Check the interfaces are satisfied
+var _ swift.Authenticator = (*auth)(nil)
diff --git a/hubic/hubic.go b/hubic/hubic.go
new file mode 100644
index 000000000..35f3047e7
--- /dev/null
+++ b/hubic/hubic.go
@@ -0,0 +1,226 @@
+// Package hubic provides an interface to the Hubic object storage
+// system.
+package hubic
+
+// This uses the normal swift mechanism to update the credentials and
+// ignores the expires field returned by the Hubic API. This may need
+// to be revisted after some actual experience.
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "time"
+
+ "github.com/ncw/rclone/fs"
+ "github.com/ncw/rclone/oauthutil"
+ "github.com/ncw/rclone/swift"
+ swiftLib "github.com/ncw/swift"
+ "golang.org/x/oauth2"
+)
+
+const (
+ rcloneClientID = "api_hubic_svWP970PvSWbw5G3PzrAqZ6X2uHeZBPI"
+ rcloneClientSecret = "8MrG3pjWyJya4OnO9ZTS4emI+9fa1ouPgvfD2MbTzfDYvO/H5czFxsTXtcji4/Hz3snz8/CrzMzlxvP9//Ty/Q=="
+)
+
+// Globals
+var (
+ // Description of how to auth for this app
+ oauthConfig = &oauth2.Config{
+ Scopes: []string{
+ "credentials.r", // Read Openstack credentials
+ },
+ Endpoint: oauth2.Endpoint{
+ AuthURL: "https://api.hubic.com/oauth/auth/",
+ TokenURL: "https://api.hubic.com/oauth/token/",
+ },
+ ClientID: rcloneClientID,
+ ClientSecret: fs.Reveal(rcloneClientSecret),
+ RedirectURL: oauthutil.RedirectLocalhostURL,
+ }
+)
+
+// Register with Fs
+func init() {
+ fs.Register(&fs.Info{
+ Name: "hubic",
+ NewFs: NewFs,
+ Config: func(name string) {
+ err := oauthutil.Config(name, oauthConfig)
+ if err != nil {
+ log.Fatalf("Failed to configure token: %v", err)
+ }
+ },
+ Options: []fs.Option{{
+ Name: oauthutil.ConfigClientID,
+ Help: "Hubic Client Id - leave blank normally.",
+ }, {
+ Name: oauthutil.ConfigClientSecret,
+ Help: "Hubic Client Secret - leave blank normally.",
+ }},
+ })
+}
+
+// credentials is the JSON returned from the Hubic API to read the
+// OpenStack credentials
+type credentials struct {
+ Token string `json:"token"` // Openstack token
+ Endpoint string `json:"endpoint"` // Openstack endpoint
+ Expires string `json:"expires"` // Expires date - eg "2015-11-09T14:24:56+01:00"
+}
+
+// Fs represents a remote hubic
+type Fs struct {
+ fs.Fs // wrapped Fs
+ client *http.Client // client for oauth api
+ credentials credentials // returned from the Hubic API
+ expires time.Time // time credentials expire
+}
+
+// Object describes a swift object
+type Object struct {
+ *swift.Object
+}
+
+// Return a string version
+func (o *Object) String() string {
+ if o == nil {
+ return ""
+ }
+ return o.Object.String()
+}
+
+// ------------------------------------------------------------
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+ if f.Fs == nil {
+ return "Hubic"
+ }
+ return fmt.Sprintf("Hubic %s", f.Fs.String())
+}
+
+// checkClose is a utility function used to check the return from
+// Close in a defer statement.
+func checkClose(c io.Closer, err *error) {
+ cerr := c.Close()
+ if *err == nil {
+ *err = cerr
+ }
+}
+
+// getCredentials reads the OpenStack Credentials using the Hubic API
+//
+// The credentials are read into the Fs
+func (f *Fs) getCredentials() (err error) {
+ req, err := http.NewRequest("GET", "https://api.hubic.com/1.0/account/credentials", nil)
+ if err != nil {
+ return err
+ }
+ req.Header.Add("User-Agent", fs.UserAgent)
+ resp, err := f.client.Do(req)
+ if err != nil {
+ return err
+ }
+ defer checkClose(resp.Body, &err)
+ if resp.StatusCode < 200 || resp.StatusCode > 299 {
+ return fmt.Errorf("Failed to get credentials: %s", resp.Status)
+ }
+ decoder := json.NewDecoder(resp.Body)
+ var result credentials
+ err = decoder.Decode(&result)
+ if err != nil {
+ return err
+ }
+ // fs.Debug(f, "Got credentials %+v", result)
+ if result.Token == "" || result.Endpoint == "" || result.Expires == "" {
+ return fmt.Errorf("Couldn't read token, result and expired from credentials")
+ }
+ f.credentials = result
+ expires, err := time.Parse(time.RFC3339, result.Expires)
+ if err != nil {
+ return err
+ }
+ f.expires = expires
+ fs.Debug(f, "Got swift credentials (expiry %v in %v)", f.expires, f.expires.Sub(time.Now()))
+ return nil
+}
+
+// NewFs constructs an Fs from the path, container:path
+func NewFs(name, root string) (fs.Fs, error) {
+ client, err := oauthutil.NewClient(name, oauthConfig)
+ if err != nil {
+ return nil, fmt.Errorf("Failed to configure Hubic: %v", err)
+ }
+
+ f := &Fs{
+ client: client,
+ }
+
+ // Make the swift Connection
+ c := &swiftLib.Connection{
+ Auth: newAuth(f),
+ UserAgent: fs.UserAgent,
+ ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
+ Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport
+ Transport: fs.Config.Transport(),
+ }
+ err = c.Authenticate()
+ if err != nil {
+ return nil, fmt.Errorf("Error authenticating swift connection: %v", err)
+ }
+
+ // Make inner swift Fs from the connection
+ swiftFs, err := swift.NewFsWithConnection(name, root, c)
+ if err != nil {
+ return nil, err
+ }
+ f.Fs = swiftFs
+ return f, nil
+}
+
+// Purge deletes all the files and the container
+//
+// Optional interface: Only implement this if you have a way of
+// deleting all the files quicker than just running Remove() on the
+// result of List()
+func (f *Fs) Purge() error {
+ fPurge, ok := f.Fs.(fs.Purger)
+ if !ok {
+ return fs.ErrorCantPurge
+ }
+ return fPurge.Purge()
+}
+
+// Copy src to this remote using server side copy operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
+ fCopy, ok := f.Fs.(fs.Copier)
+ if !ok {
+ return nil, fs.ErrorCantCopy
+ }
+ return fCopy.Copy(src, remote)
+}
+
+// UnWrap returns the Fs that this Fs is wrapping
+func (f *Fs) UnWrap() fs.Fs {
+ return f.Fs
+}
+
+// Check the interfaces are satisfied
+var (
+ _ fs.Fs = (*Fs)(nil)
+ _ fs.Purger = (*Fs)(nil)
+ _ fs.Copier = (*Fs)(nil)
+ _ fs.UnWrapper = (*Fs)(nil)
+)
diff --git a/hubic/hubic_test.go b/hubic/hubic_test.go
new file mode 100644
index 000000000..84e344dc4
--- /dev/null
+++ b/hubic/hubic_test.go
@@ -0,0 +1,56 @@
+// Test Hubic filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: make gen_tests
+package hubic_test
+
+import (
+ "testing"
+
+ "github.com/ncw/rclone/fs"
+ "github.com/ncw/rclone/fstest/fstests"
+ "github.com/ncw/rclone/hubic"
+)
+
+func init() {
+ fstests.NilObject = fs.Object((*hubic.Object)(nil))
+ fstests.RemoteName = "TestHubic:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T) { fstests.TestInit(t) }
+func TestFsString(t *testing.T) { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
+func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
+func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
+func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
+func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
+func TestFsMove(t *testing.T) { fstests.TestFsMove(t) }
+func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) }
+func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
+func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
+func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
+func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
diff --git a/make_manual.py b/make_manual.py
index 7cb9ed927..e14309b69 100755
--- a/make_manual.py
+++ b/make_manual.py
@@ -25,6 +25,7 @@ docs = [
"googlecloudstorage.md",
"amazonclouddrive.md",
"onedrive.md",
+ "hubic.md",
"local.md",
"changelog.md",
"bugs.md",
diff --git a/rclone.go b/rclone.go
index ae6784020..878ddfb50 100644
--- a/rclone.go
+++ b/rclone.go
@@ -20,6 +20,7 @@ import (
_ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/googlecloudstorage"
+ _ "github.com/ncw/rclone/hubic"
_ "github.com/ncw/rclone/local"
_ "github.com/ncw/rclone/onedrive"
_ "github.com/ncw/rclone/s3"
diff --git a/swift/swift.go b/swift/swift.go
index 63ad5ba7b..6e98c03ff 100644
--- a/swift/swift.go
+++ b/swift/swift.go
@@ -159,16 +159,13 @@ func swiftConnection(name string) (*swift.Connection, error) {
return c, nil
}
-// NewFs contstructs an Fs from the path, container:path
-func NewFs(name, root string) (fs.Fs, error) {
+// NewFsWithConnection contstructs an Fs from the path, container:path
+// and authenticated connection
+func NewFsWithConnection(name, root string, c *swift.Connection) (fs.Fs, error) {
container, directory, err := parsePath(root)
if err != nil {
return nil, err
}
- c, err := swiftConnection(name)
- if err != nil {
- return nil, err
- }
f := &Fs{
name: name,
c: *c,
@@ -196,6 +193,15 @@ func NewFs(name, root string) (fs.Fs, error) {
return f, nil
}
+// NewFs contstructs an Fs from the path, container:path
+func NewFs(name, root string) (fs.Fs, error) {
+ c, err := swiftConnection(name)
+ if err != nil {
+ return nil, err
+ }
+ return NewFsWithConnection(name, root, c)
+}
+
// Return an FsObject from a path
//
// May return nil if an error occurred