Implement pcloud remote - #418

CONTRIBUTING.md

@@ -241,10 +241,11 @@ Research
 Getting going
 
   * Create `remote/remote.go` (copy this from a similar remote)
-    * onedrive is a good one to start from if you have a directory based remote
+    * box is a good one to start from if you have a directory based remote
     * b2 is a good one to start from if you have a bucket based remote
   * Add your remote to the imports in `fs/all/all.go`
   * HTTP based remotes are easiest to maintain if they use rclone's rest module, but if there is a really good go SDK then use that instead.
+  * Try to implement as many optional methods as possible as it makes the remote more usable.
 
 Unit tests
 

README.md

@@ -28,6 +28,7 @@ Rclone is a command line program to sync files and directories to and from
   * Microsoft Azure Blob Storage
   * Microsoft OneDrive
   * Openstack Swift / Rackspace cloud files / Memset Memstore / OVH / Oracle Cloud Storage
+  * pCloud
   * QingStor
   * SFTP
   * Yandex Disk

bin/make_manual.py

@@ -36,6 +36,7 @@ docs = [
     "onedrive.md",
     "qingstor.md",
     "swift.md",
+    "pcloud.md",
     "sftp.md",
     "yandex.md",
 

cmd/cmd.go

@@ -54,6 +54,7 @@ from various cloud storage systems and using file transfer services, such as:
   * Microsoft Azure Blob Storage
   * Microsoft OneDrive
   * Openstack Swift / Rackspace cloud files / Memset Memstore
+  * pCloud
   * QingStor
   * SFTP
   * Yandex Disk

docs/content/about.md

@@ -31,6 +31,7 @@ Rclone is a command line program to sync files and directories to and from:
 * {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}}
 * {{< provider name="OVH" home="https://www.ovh.co.uk/public-cloud/storage/object-storage/" config="/swift/" >}}
 * {{< provider name="Openstack Swift" home="https://docs.openstack.org/swift/latest/" config="/swift/" >}}
+* {{< provider name="pCloud" home="https://www.pcloud.com/" config="/pcloud/" >}}
 * {{< provider name="Oracle Cloud Storage" home="https://cloud.oracle.com/storage-opc" config="/swift/" >}}
 * {{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}}
 * {{< provider name="Rackspace Cloud Files" home="https://www.rackspace.com/cloud/files" config="/swift/" >}}

docs/content/docs.md

@@ -33,6 +33,7 @@ See the following for detailed instructions for
   * [Microsoft Azure Blob Storage](/azureblob/)
   * [Microsoft OneDrive](/onedrive/)
   * [Openstack Swift / Rackspace Cloudfiles / Memset Memstore](/swift/)
+  * [Pcloud](/pcloud/)
   * [QingStor](/qingstor/)
   * [SFTP](/sftp/)
   * [Yandex Disk](/yandex/)

docs/content/overview.md

@@ -30,6 +30,7 @@ Here is an overview of the major features of each cloud storage system.
 | Microsoft Azure Blob Storage | MD5         | Yes     | No               | No              | R/W       |
 | Microsoft OneDrive           | SHA1        | Yes     | Yes              | No              | R         |
 | Openstack Swift              | MD5         | Yes     | No               | No              | R/W       |
+| pCloud                       | MD5, SHA1   | Yes     | No               | No              | W         |
 | QingStor                     | MD5         | No      | No               | No              | R/W       |
 | SFTP                         | MD5, SHA1 ‡ | Yes     | Depends          | No              | -         |
 | Yandex Disk                  | MD5         | Yes     | No               | No              | R/W       |
@@ -130,6 +131,7 @@ operations more efficient.
 | Microsoft Azure Blob Storage | Yes   | Yes  | No   | No      | No      | Yes   | No           |
 | Microsoft OneDrive           | Yes   | Yes  | Yes  | No [#197](https://github.com/ncw/rclone/issues/197) | No [#575](https://github.com/ncw/rclone/issues/575) | No | No |
 | Openstack Swift              | Yes † | Yes  | No   | No      | No      | Yes   | Yes          |
+| pCloud                       | Yes   | Yes  | Yes  | Yes     | Yes     | No    | No           |
 | QingStor                     | No    | Yes  | No   | No      | No      | Yes   | No           |
 | SFTP                         | No    | No   | Yes  | Yes     | No      | No    | Yes          |
 | Yandex Disk                  | Yes   | No   | No   | No      | Yes     | Yes   | Yes          |

docs/content/pcloud.md

@@ -0,0 +1,135 @@
+---
+title: "pCloud"
+description: "Rclone docs for pCloud"
+date: "2017-10-01"
+---
+
+<i class="fa fa-cloud"></i> pCloud
+-----------------------------------------
+
+Paths are specified as `remote:path`
+
+Paths may be as deep as required, eg `remote:directory/subdirectory`.
+
+The initial setup for pCloud involves getting a token from pCloud which you
+need to do in your browser.  `rclone config` walks you through it.
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Box
+   \ "box"
+ 5 / Dropbox
+   \ "dropbox"
+ 6 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 7 / FTP Connection
+   \ "ftp"
+ 8 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 9 / Google Drive
+   \ "drive"
+10 / Hubic
+   \ "hubic"
+11 / Local Disk
+   \ "local"
+12 / Microsoft Azure Blob Storage
+   \ "azureblob"
+13 / Microsoft OneDrive
+   \ "onedrive"
+14 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+15 / Pcloud
+   \ "pcloud"
+16 / QingClound Object Storage
+   \ "qingstor"
+17 / SSH/SFTP Connection
+   \ "sftp"
+18 / Yandex Disk
+   \ "yandex"
+19 / http Connection
+   \ "http"
+Storage> pcloud
+Pcloud App Client Id - leave blank normally.
+client_id> 
+Pcloud App Client Secret - leave blank normally.
+client_secret> 
+Remote config
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+client_id = 
+client_secret = 
+token = {"access_token":"XXX","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+See the [remote setup docs](/remote_setup/) for how to set it up on a
+machine with no Internet browser available.
+
+Note that rclone runs a webserver on your local machine to collect the
+token as returned from pCloud. This only runs from the moment it opens
+your browser to the moment you get back the verification code.  This
+is on `http://127.0.0.1:53682/` and this it may require you to unblock
+it temporarily if you are running a host firewall.
+
+Once configured you can then use `rclone` like this,
+
+List directories in top level of your pCloud
+
+    rclone lsd remote:
+
+List all the files in your pCloud
+
+    rclone ls remote:
+
+To copy a local directory to an pCloud directory called backup
+
+    rclone copy /home/source remote:backup
+
+### Modified time and hashes ###
+
+pCloud allows modification times to be set on objects accurate to 1
+second.  These will be used to detect whether objects need syncing or
+not.  In order to set a Modification time pCloud requires the object
+be re-uploaded.
+
+pCloud supports MD5 and SHA1 type hashes, so you can use the
+`--checksum` flag.
+
+### Deleting files ###
+
+Deleted files will be moved to the trash.  Your subscription level
+will determine how long items stay in the trash.  `rclone cleanup` can
+be used to empty the trash.

docs/layouts/chrome/navbar.html

@@ -64,6 +64,7 @@
                     <li><a href="/onedrive/"><i class="fa fa-windows"></i> Microsoft OneDrive</a></li>
                     <li><a href="/qingstor/"><i class="fa fa-hdd-o"></i> QingStor</a></li>
                     <li><a href="/swift/"><i class="fa fa-space-shuttle"></i> Openstack Swift</a></li>
+                    <li><a href="/pcloud/"><i class="fa fa-cloud"></i> pCloud</a></li>
                     <li><a href="/sftp/"><i class="fa fa-server"></i> SFTP</a></li>
                     <li><a href="/yandex/"><i class="fa fa-space-shuttle"></i> Yandex Disk</a></li>
                     <li><a href="/local/"><i class="fa fa-file"></i> The local filesystem</a></li>

fs/all/all.go

@@ -15,6 +15,7 @@ import (
 	_ "github.com/ncw/rclone/hubic"
 	_ "github.com/ncw/rclone/local"
 	_ "github.com/ncw/rclone/onedrive"
+	_ "github.com/ncw/rclone/pcloud"
 	_ "github.com/ncw/rclone/qingstor"
 	_ "github.com/ncw/rclone/s3"
 	_ "github.com/ncw/rclone/sftp"

fs/test_all.go

@@ -113,6 +113,11 @@ var (
 			SubDir:   true,
 			FastList: true,
 		},
+		{
+			Name:     "TestPcloud:",
+			SubDir:   false,
+			FastList: false,
+		},
 	}
 	binary = "fs.test"
 	// Flags

fstest/fstests/gen_tests.go

@@ -164,5 +164,6 @@ func main() {
 	generateTestProgram(t, fns, "Box")
 	generateTestProgram(t, fns, "QingStor", buildConstraint("!plan9"))
 	generateTestProgram(t, fns, "AzureBlob", buildConstraint("go1.7"))
+	generateTestProgram(t, fns, "Pcloud")
 	log.Printf("Done")
 }

pcloud/api/types.go

@@ -0,0 +1,153 @@
+// Package api has type definitions for pcloud
+//
+// Converted from the API docs with help from https://mholt.github.io/json-to-go/
+package api
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	// Sun, 16 Mar 2014 17:26:04 +0000
+	timeFormat = `"` + time.RFC1123Z + `"`
+)
+
+// Time represents represents date and time information for the
+// pcloud API, by using RFC1123Z
+type Time time.Time
+
+// MarshalJSON turns a Time into JSON (in UTC)
+func (t *Time) MarshalJSON() (out []byte, err error) {
+	timeString := (*time.Time)(t).Format(timeFormat)
+	return []byte(timeString), nil
+}
+
+// UnmarshalJSON turns JSON into a Time
+func (t *Time) UnmarshalJSON(data []byte) error {
+	newT, err := time.Parse(timeFormat, string(data))
+	if err != nil {
+		return err
+	}
+	*t = Time(newT)
+	return nil
+}
+
+// Error is returned from pcloud when things go wrong
+//
+// If result is 0 then everything is OK
+type Error struct {
+	Result      int    `json:"result"`
+	ErrorString string `json:"error"`
+}
+
+// Error returns a string for the error and statistifes the error interface
+func (e *Error) Error() string {
+	return fmt.Sprintf("pcloud error: %s (%d)", e.ErrorString, e.Result)
+}
+
+// Update returns err directly if it was != nil, otherwise it returns
+// an Error or nil if no error was detected
+func (e *Error) Update(err error) error {
+	if err != nil {
+		return err
+	}
+	if e.Result == 0 {
+		return nil
+	}
+	return e
+}
+
+// Check Error statisfies the error interface
+var _ error = (*Error)(nil)
+
+// Item describes a folder or a file as returned by Get Folder Items and others
+type Item struct {
+	Path           string `json:"path"`
+	Name           string `json:"name"`
+	Created        Time   `json:"created"`
+	IsMine         bool   `json:"ismine"`
+	Thumb          bool   `json:"thumb"`
+	Modified       Time   `json:"modified"`
+	Comments       int    `json:"comments"`
+	ID             string `json:"id"`
+	IsShared       bool   `json:"isshared"`
+	IsDeleted      bool   `json:"isdeleted"`
+	Icon           string `json:"icon"`
+	IsFolder       bool   `json:"isfolder"`
+	ParentFolderID int64  `json:"parentfolderid"`
+	FolderID       int64  `json:"folderid,omitempty"`
+	Height         int    `json:"height,omitempty"`
+	FileID         int64  `json:"fileid,omitempty"`
+	Width          int    `json:"width,omitempty"`
+	Hash           uint64 `json:"hash,omitempty"`
+	Category       int    `json:"category,omitempty"`
+	Size           int64  `json:"size,omitempty"`
+	ContentType    string `json:"contenttype,omitempty"`
+	Contents       []Item `json:"contents"`
+}
+
+// ModTime returns the modification time of the item
+func (i *Item) ModTime() (t time.Time) {
+	t = time.Time(i.Modified)
+	if t.IsZero() {
+		t = time.Time(i.Created)
+	}
+	return t
+}
+
+// ItemResult is returned from the /listfolder, /createfolder, /deletefolder, /deletefile etc methods
+type ItemResult struct {
+	Error
+	Metadata Item `json:"metadata"`
+}
+
+// Hashes contains the supported hashes
+type Hashes struct {
+	SHA1 string `json:"sha1"`
+	MD5  string `json:"md5"`
+}
+
+// UploadFileResponse is the response from /uploadfile
+type UploadFileResponse struct {
+	Error
+	Items     []Item   `json:"metadata"`
+	Checksums []Hashes `json:"checksums"`
+	Fileids   []int64  `json:"fileids"`
+}
+
+// GetFileLinkResult is returned from /getfilelink
+type GetFileLinkResult struct {
+	Error
+	Dwltag  string   `json:"dwltag"`
+	Hash    uint64   `json:"hash"`
+	Size    int64    `json:"size"`
+	Expires Time     `json:"expires"`
+	Path    string   `json:"path"`
+	Hosts   []string `json:"hosts"`
+}
+
+// IsValid returns whether the link is valid and has not expired
+func (g *GetFileLinkResult) IsValid() bool {
+	if g == nil {
+		return false
+	}
+	if len(g.Hosts) == 0 {
+		return false
+	}
+	return time.Until(time.Time(g.Expires)) > 30*time.Second
+}
+
+// URL returns a URL from the Path and Hosts.  Check with IsValid
+// before calling.
+func (g *GetFileLinkResult) URL() string {
+	// FIXME rotate the hosts?
+	return "https://" + g.Hosts[0] + g.Path
+}
+
+// ChecksumFileResult is returned from /checksumfile
+type ChecksumFileResult struct {
+	Error
+	Hashes
+	Metadata Item `json:"metadata"`
+}

pcloud/pcloud_test.go

@@ -0,0 +1,73 @@
+// Test Pcloud filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: make gen_tests
+package pcloud_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/pcloud"
+)
+
+func TestSetup(t *testing.T) {
+	fstests.NilObject = fs.Object((*pcloud.Object)(nil))
+	fstests.RemoteName = "TestPcloud:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                { fstests.TestInit(t) }
+func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
+func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
+func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
+func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
+func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
+func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
+func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
+func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
+func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
+func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
+func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
+func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
+func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
+func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
+func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
+func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
+func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
+func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
+func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
+func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
+func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
+func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
+func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
+func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
+func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
+func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
+func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
+func TestFsDirChangeNotify(t *testing.T)   { fstests.TestFsDirChangeNotify(t) }
+func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
+func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
+func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
+func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
+func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
+func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
+func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
+func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
+func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
+func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
+func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
+func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
+func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }