diff --git a/README.md b/README.md index 81750d0c0..1839e8c5f 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Rclone is a command line program to sync files and directories to and from * Amazon Cloud Drive * Microsoft One Drive * Hubic + * Backblaze B2 * The local filesystem Features diff --git a/b2/api/types.go b/b2/api/types.go new file mode 100644 index 000000000..9221c2533 --- /dev/null +++ b/b2/api/types.go @@ -0,0 +1,147 @@ +package api + +import ( + "fmt" + "strconv" + "time" +) + +// Error describes a B2 error response +type Error struct { + Status int `json:"status"` // The numeric HTTP status code. Always matches the status in the HTTP response. + Code string `json:"code"` // A single-identifier code that identifies the error. + Message string `json:"message"` // A human-readable message, in English, saying what went wrong. +} + +// Error statisfies the error interface +func (e *Error) Error() string { + return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code) +} + +// Account describes a B2 account +type Account struct { + ID string `json:"accountId"` // The identifier for the account. +} + +// Bucket describes a B2 bucket +type Bucket struct { + ID string `json:"bucketId"` + AccountID string `json:"accountId"` + Name string `json:"bucketName"` + Type string `json:"bucketType"` +} + +// Timestamp is a UTC time when this file was uploaded. It is a base +// 10 number of milliseconds since midnight, January 1, 1970 UTC. This +// fits in a 64 bit integer such as the type "long" in the programming +// language Java. It is intended to be compatible with Java's time +// long. For example, it can be passed directly into the java call +// Date.setTime(long time). +type Timestamp time.Time + +// MarshalJSON turns a Timestamp into JSON (in UTC) +func (t *Timestamp) MarshalJSON() (out []byte, err error) { + timestamp := (*time.Time)(t).UTC().UnixNano() + return []byte(strconv.FormatInt(timestamp/1E6, 10)), nil +} + +// UnmarshalJSON turns JSON into a Timestamp +func (t *Timestamp) UnmarshalJSON(data []byte) error { + timestamp, err := strconv.ParseInt(string(data), 10, 64) + if err != nil { + return err + } + *t = Timestamp(time.Unix(timestamp/1E3, (timestamp%1E3)*1E6)) + return nil +} + +// File is info about a file +type File struct { + ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version. + Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name. + Action string `json:"action"` // Either "upload" or "hide". "upload" means a file that was uploaded to B2 Cloud Storage. "hide" means a file version marking the file as hidden, so that it will not show up in b2_list_file_names. The result of b2_list_file_names will contain only "upload". The result of b2_list_file_versions may have both. + Size int64 `json:"size"` // The number of bytes in the file. + UploadTimestamp Timestamp `json:"uploadTimestamp"` // This is a UTC time when this file was uploaded. +} + +// AuthorizeAccountResponse is as returned from the b2_authorize_account call +type AuthorizeAccountResponse struct { + AccountID string `json:"accountId"` // The identifier for the account. + AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header. + APIURL string `json:"apiUrl"` // The base URL to use for all API calls except for uploading and downloading files. + DownloadURL string `json:"downloadUrl"` // The base URL to use for downloading files. +} + +// ListBucketsResponse is as returned from the b2_list_buckets call +type ListBucketsResponse struct { + Buckets []Bucket `json:"buckets"` +} + +// ListFileNamesRequest is as passed to b2_list_file_names or b2_list_file_versions +type ListFileNamesRequest struct { + BucketID string `json:"bucketId"` // required - The bucket to look for file names in. + StartFileName string `json:"startFileName,omitempty"` // optional - The first file name to return. If there is a file with this name, it will be returned in the list. If not, the first file name after this the first one after this name. + MaxFileCount int `json:"maxFileCount,omitempty"` // optional - The maximum number of files to return from this call. The default value is 100, and the maximum allowed is 1000. + StartFileID string `json:"startFileId,omitempty"` // optional - What to pass in to startFileId for the next search to continue where this one left off. +} + +// ListFileNamesResponse is as received from b2_list_file_names or b2_list_file_versions +type ListFileNamesResponse struct { + Files []File `json:"files"` // An array of objects, each one describing one file. + NextFileName *string `json:"nextFileName"` // What to pass in to startFileName for the next search to continue where this one left off, or null if there are no more files. + NextFileID *string `json:"nextFileId"` // What to pass in to startFileId for the next search to continue where this one left off, or null if there are no more files. +} + +// GetUploadURLRequest is passed to b2_get_upload_url +type GetUploadURLRequest struct { + BucketID string `json:"bucketId"` // The ID of the bucket that you want to upload to. +} + +// GetUploadURLResponse is received from b2_get_upload_url +type GetUploadURLResponse struct { + BucketID string `json:"bucketId"` // The unique ID of the bucket. + UploadURL string `json:"uploadUrl"` // The URL that can be used to upload files to this bucket, see b2_upload_file. + AuthorizationToken string `json:"authorizationToken"` // The authorizationToken that must be used when uploading files to this bucket, see b2_upload_file. +} + +// FileInfo is received from b2_upload_file and b2_get_file_info +type FileInfo struct { + ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version. + Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name. + AccountID string `json:"accountId"` // Your account ID. + BucketID string `json:"bucketId"` // The bucket that the file is in. + Size int64 `json:"contentLength"` // The number of bytes stored in the file. + SHA1 string `json:"contentSha1"` // The SHA1 of the bytes stored in the file. + ContentType string `json:"contentType"` // The MIME type of the file. + Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file. +} + +// CreateBucketRequest is used to create a bucket +type CreateBucketRequest struct { + AccountID string `json:"accountId"` + Name string `json:"bucketName"` + Type string `json:"bucketType"` +} + +// DeleteBucketRequest is used to create a bucket +type DeleteBucketRequest struct { + ID string `json:"bucketId"` + AccountID string `json:"accountId"` +} + +// DeleteFileRequest is used to delete a file version +type DeleteFileRequest struct { + ID string `json:"fileId"` // The ID of the file, as returned by b2_upload_file, b2_list_file_names, or b2_list_file_versions. + Name string `json:"fileName"` // The name of this file. +} + +// HideFileRequest is used to delete a file +type HideFileRequest struct { + BucketID string `json:"bucketId"` // The bucket containing the file to hide. + Name string `json:"fileName"` // The name of the file to hide. +} + +// GetFileInfoRequest is used to return a FileInfo struct with b2_get_file_info +type GetFileInfoRequest struct { + ID string `json:"fileId"` // The ID of the file, as returned by b2_upload_file, b2_list_file_names, or b2_list_file_versions. +} diff --git a/b2/b2.go b/b2/b2.go new file mode 100644 index 000000000..b5322629c --- /dev/null +++ b/b2/b2.go @@ -0,0 +1,972 @@ +// Package b2 provides an interface to the Backblaze B2 object storage system +package b2 + +// FIXME if b2 could set the mod time then it has everything else to +// implement mod times. It is just missing that bit of API. + +import ( + "bytes" + "crypto/sha1" + "errors" + "fmt" + "hash" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/ncw/rclone/b2/api" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/rest" +) + +const ( + defaultEndpoint = "https://api.backblaze.com" + headerPrefix = "x-bz-info-" // lower case as that is what the server returns + timeKey = "src_last_modified_millis" + timeHeader = headerPrefix + timeKey + sha1Header = "X-Bz-Content-Sha1" +) + +// Register with Fs +func init() { + fs.Register(&fs.Info{ + Name: "b2", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "account", + Help: "Account ID", + }, { + Name: "key", + Help: "Application Key", + }, { + Name: "endpoint", + Help: "Endpoint for the service - leave blank normally.", + }, + }, + }) +} + +// Fs represents a remote b2 server +type Fs struct { + name string // name of this remote + srv *rest.Client // the connection to the b2 server + bucket string // the bucket we are working on + bucketIDMutex sync.Mutex // mutex to protect _bucketID + _bucketID string // the ID of the bucket we are working on + root string // the path we are working on if any + info api.AuthorizeAccountResponse // result of authorize call + uploadMu sync.Mutex // lock for upload variable + upload api.GetUploadURLResponse // result of get upload URL call +} + +// Object describes a b2 object +// +// Will definitely have info +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + info api.File // Info from the b2 object if known + modTime time.Time // The modified time of the object if known +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + if f.root == "" { + return f.bucket + } + return f.bucket + "/" + f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + if f.root == "" { + return fmt.Sprintf("B2 bucket %s", f.bucket) + } + return fmt.Sprintf("B2 bucket %s path %s", f.bucket, f.root) +} + +// Pattern to match a b2 path +var matcher = regexp.MustCompile(`^([^/]*)(.*)$`) + +// parseParse parses a b2 'url' +func parsePath(path string) (bucket, directory string, err error) { + parts := matcher.FindStringSubmatch(path) + if parts == nil { + err = fmt.Errorf("Couldn't find bucket in b2 path %q", path) + } else { + bucket, directory = parts[1], parts[2] + directory = strings.Trim(directory, "/") + } + return +} + +// errorHandler parses a non 2xx error response into an error +func errorHandler(resp *http.Response) error { + // Decode error response + errResponse := new(api.Error) + err := rest.DecodeJSON(resp, &errResponse) + if err != nil { + fs.Debug(nil, "Couldn't decode error response: %v", err) + } + if errResponse.Code == "" { + errResponse.Code = "unknown" + } + if errResponse.Status == 0 { + errResponse.Status = resp.StatusCode + } + if errResponse.Message == "" { + errResponse.Message = "Unknown " + resp.Status + } + return errResponse +} + +// NewFs contstructs an Fs from the path, bucket:path +func NewFs(name, root string) (fs.Fs, error) { + bucket, directory, err := parsePath(root) + if err != nil { + return nil, err + } + f := &Fs{ + name: name, + bucket: bucket, + root: directory, + } + + account := fs.ConfigFile.MustValue(name, "account") + if account == "" { + return nil, errors.New("account not found") + } + key := fs.ConfigFile.MustValue(name, "key") + if key == "" { + return nil, errors.New("key not found") + } + endpoint := fs.ConfigFile.MustValue(name, "endpoint", defaultEndpoint) + + f.srv = rest.NewClient(fs.Config.Client()).SetRoot(endpoint + "/b2api/v1").SetErrorHandler(errorHandler) + + opts := rest.Opts{ + Method: "GET", + Path: "/b2_authorize_account", + UserName: account, + Password: key, + } + _, err = f.srv.CallJSON(&opts, nil, &f.info) + if err != nil { + return nil, fmt.Errorf("Failed to authenticate: %v", err) + } + f.srv.SetRoot(f.info.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken) + + if f.root != "" { + f.root += "/" + // Check to see if the (bucket,directory) is actually an existing file + oldRoot := f.root + remote := path.Base(directory) + f.root = path.Dir(directory) + if f.root == "." { + f.root = "" + } else { + f.root += "/" + } + obj := f.NewFsObject(remote) + if obj != nil { + return fs.NewLimited(f, obj), nil + } + f.root = oldRoot + } + return f, nil +} + +// getUploadURL returns the UploadURL and the AuthorizationToken +func (f *Fs) getUploadURL() (string, string, error) { + f.uploadMu.Lock() + defer f.uploadMu.Unlock() + bucketID, err := f.getBucketID() + if err != nil { + return "", "", err + } + if f.upload.UploadURL == "" || f.upload.AuthorizationToken == "" { + opts := rest.Opts{ + Method: "POST", + Path: "/b2_get_upload_url", + } + var request = api.GetUploadURLRequest{ + BucketID: bucketID, + } + _, err := f.srv.CallJSON(&opts, &request, &f.upload) + if err != nil { + return "", "", fmt.Errorf("Failed to get upload URL: %v", err) + } + } + return f.upload.UploadURL, f.upload.AuthorizationToken, nil +} + +// clearUploadURL clears the current UploadURL and the AuthorizationToken +func (f *Fs) clearUploadURL() { + f.uploadMu.Lock() + f.upload = api.GetUploadURLResponse{} + defer f.uploadMu.Unlock() +} + +// Return an FsObject from a path +// +// May return nil if an error occurred +func (f *Fs) newFsObjectWithInfo(remote string, info *api.File) fs.Object { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + // Set info but not headers + o.info = *info + } else { + err := o.readMetaData() // reads info and headers, returning an error + if err != nil { + fs.Debug(o, "Failed to read metadata: %s", err) + return nil + } + } + return o +} + +// NewFsObject returns an FsObject from a path +// +// May return nil if an error occurred +func (f *Fs) NewFsObject(remote string) fs.Object { + return f.newFsObjectWithInfo(remote, nil) +} + +// listFn is called from list to handle an object +type listFn func(string, *api.File) error + +// list lists the objects into the function supplied from +// the bucket and root supplied +// +// If prefix is set then startFileName is used as a prefix which all +// files must have +// +// If limit is > 0 then it limits to that many files (must be less +// than 1000) +// +// If hidden is set then it will list the hidden (deleted) files too. +func (f *Fs) list(prefix string, limit int, hidden bool, fn listFn) error { + bucketID, err := f.getBucketID() + if err != nil { + return err + } + chunkSize := 1000 + if limit > 0 { + chunkSize = limit + } + var request = api.ListFileNamesRequest{ + BucketID: bucketID, + MaxFileCount: chunkSize, + } + prefix = f.root + prefix + if prefix != "" { + request.StartFileName = prefix + } + var response api.ListFileNamesResponse + opts := rest.Opts{ + Method: "POST", + Path: "/b2_list_file_names", + } + if hidden { + opts.Path = "/b2_list_file_versions" + } + for { + _, err = f.srv.CallJSON(&opts, &request, &response) + if err != nil { + return err + } + for i := range response.Files { + file := &response.Files[i] + // Finish if file name no longer has prefix + if !strings.HasPrefix(file.Name, prefix) { + return nil + } + err = fn(file.Name[len(prefix):], file) + if err != nil { + return err + } + } + // end if no NextFileName + if response.NextFileName == nil { + break + } + request.StartFileName = *response.NextFileName + if response.NextFileID != nil { + request.StartFileID = *response.NextFileID + } + } + return nil +} + +// List walks the path returning a channel of FsObjects +func (f *Fs) List() fs.ObjectsChan { + out := make(fs.ObjectsChan, fs.Config.Checkers) + if f.bucket == "" { + // Return no objects at top level list + close(out) + fs.Stats.Error() + fs.ErrorLog(f, "Can't list objects at root - choose a bucket using lsd") + } else { + // List the objects + go func() { + defer close(out) + err := f.list("", 0, false, func(remote string, object *api.File) error { + if o := f.newFsObjectWithInfo(remote, object); o != nil { + out <- o + } + return nil + }) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "Couldn't list bucket %q: %s", f.bucket, err) + } + }() + } + return out +} + +// listBucketFn is called from listBuckets to handle a bucket +type listBucketFn func(*api.Bucket) + +// listBuckets lists the buckets to the function supplied +func (f *Fs) listBuckets(fn listBucketFn) error { + var account = api.Account{ID: f.info.AccountID} + var response api.ListBucketsResponse + opts := rest.Opts{ + Method: "POST", + Path: "/b2_list_buckets", + } + _, err := f.srv.CallJSON(&opts, &account, &response) + if err != nil { + return err + } + for i := range response.Buckets { + fn(&response.Buckets[i]) + } + return nil +} + +// getBucketID finds the ID for the current bucket name +func (f *Fs) getBucketID() (bucketID string, err error) { + f.bucketIDMutex.Lock() + defer f.bucketIDMutex.Unlock() + if f._bucketID != "" { + return f._bucketID, nil + } + err = f.listBuckets(func(bucket *api.Bucket) { + if bucket.Name == f.bucket { + bucketID = bucket.ID + } + }) + if bucketID == "" { + err = fmt.Errorf("Couldn't find bucket %q", f.bucket) + } + f._bucketID = bucketID + return bucketID, err +} + +// setBucketID sets the ID for the current bucket name +func (f *Fs) setBucketID(ID string) { + f.bucketIDMutex.Lock() + f._bucketID = ID + f.bucketIDMutex.Unlock() +} + +// clearBucketID clears the ID for the current bucket name +func (f *Fs) clearBucketID() { + f.bucketIDMutex.Lock() + f._bucketID = "" + f.bucketIDMutex.Unlock() +} + +// ListDir lists the buckets +func (f *Fs) ListDir() fs.DirChan { + out := make(fs.DirChan, fs.Config.Checkers) + if f.bucket == "" { + // List the buckets + go func() { + defer close(out) + err := f.listBuckets(func(bucket *api.Bucket) { + out <- &fs.Dir{ + Name: bucket.Name, + Bytes: -1, + Count: -1, + } + }) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "Error listing buckets: %v", err) + } + }() + } else { + // List the directories in the path in the bucket + go func() { + defer close(out) + lastDir := "" + err := f.list("", 0, false, func(remote string, object *api.File) error { + slash := strings.IndexRune(remote, '/') + if slash < 0 { + return nil + } + dir := remote[:slash] + if dir == lastDir { + return nil + } + out <- &fs.Dir{ + Name: dir, + Bytes: -1, + Count: -1, + } + lastDir = dir + return nil + }) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "Couldn't list bucket %q: %s", f.bucket, err) + } + }() + } + return out +} + +// Put the object into the bucket +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { + // Temporary Object under construction + fs := &Object{ + fs: f, + remote: remote, + } + return fs, fs.Update(in, modTime, size) +} + +// Mkdir creates the bucket if it doesn't exist +func (f *Fs) Mkdir() error { + opts := rest.Opts{ + Method: "POST", + Path: "/b2_create_bucket", + } + var request = api.CreateBucketRequest{ + AccountID: f.info.AccountID, + Name: f.bucket, + Type: "allPrivate", + } + var response api.Bucket + _, err := f.srv.CallJSON(&opts, &request, &response) + if err != nil { + if apiErr, ok := err.(*api.Error); ok { + if apiErr.Code == "duplicate_bucket_name" { + return nil + } + } + return fmt.Errorf("Failed to create bucket: %v", err) + } + f.setBucketID(response.ID) + return nil +} + +// Rmdir deletes the bucket if the fs is at the root +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir() error { + if f.root != "" { + return nil + } + opts := rest.Opts{ + Method: "POST", + Path: "/b2_delete_bucket", + } + bucketID, err := f.getBucketID() + if err != nil { + return err + } + var request = api.DeleteBucketRequest{ + ID: bucketID, + AccountID: f.info.AccountID, + } + var response api.Bucket + _, err = f.srv.CallJSON(&opts, &request, &response) + if err != nil { + return fmt.Errorf("Failed to delete bucket: %v", err) + } + f.clearBucketID() + f.clearUploadURL() + return nil +} + +// Precision of the remote +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// deleteByID deletes a file version given Name and ID +func (f *Fs) deleteByID(ID, Name string) error { + opts := rest.Opts{ + Method: "POST", + Path: "/b2_delete_file_version", + } + var request = api.DeleteFileRequest{ + ID: ID, + Name: Name, + } + var response api.File + _, err := f.srv.CallJSON(&opts, &request, &response) + if err != nil { + return fmt.Errorf("Failed to delete %q: %v", Name, err) + } + return nil +} + +// Purge deletes all the files and directories +// +// Implemented here so we can make sure we delete old versions. +func (f *Fs) Purge() error { + var errReturn error + var checkErrMutex sync.Mutex + var checkErr = func(err error) { + if err == nil { + return + } + checkErrMutex.Lock() + defer checkErrMutex.Unlock() + fs.Stats.Error() + fs.ErrorLog(f, "Purge error: %v", err) + if errReturn == nil { + errReturn = err + } + } + + // Delete Config.Transfers in parallel + toBeDeleted := make(chan *api.File, fs.Config.Transfers) + var wg sync.WaitGroup + wg.Add(fs.Config.Transfers) + for i := 0; i < fs.Config.Transfers; i++ { + go func() { + defer wg.Done() + for object := range toBeDeleted { + checkErr(f.deleteByID(object.ID, object.Name)) + } + }() + } + checkErr(f.list("", 0, true, func(remote string, object *api.File) error { + fs.Debug(remote, "Deleting (id %q)", object.ID) + toBeDeleted <- object + return nil + })) + close(toBeDeleted) + wg.Wait() + + checkErr(f.Rmdir()) + return errReturn +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Fs { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Md5sum returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Md5sum() (string, error) { + return "", nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.info.Size +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData() (err error) { + if o.info.ID != "" { + return nil + } + err = o.fs.list(o.remote, 1, false, func(remote string, object *api.File) error { + if remote == "" { + o.info = *object + } + return nil + }) + if o.info.ID != "" { + return nil + } + return fmt.Errorf("Object %q not found", o.remote) +} + +// timeString returns modTime as the number of milliseconds +// elapsed since January 1, 1970 UTC as a decimal string. +func timeString(modTime time.Time) string { + return strconv.FormatInt(modTime.UnixNano()/1E6, 10) +} + +// parseTimeString converts a decimal string number of milliseconds +// elapsed since January 1, 1970 UTC into a time.Time +func parseTimeString(timeString string) (result time.Time, err error) { + if timeString == "" { + return result, fmt.Errorf("%q not found in metadata", timeKey) + } + unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64) + if err != nil { + return result, err + } + return time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC(), nil +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() (result time.Time) { + if !o.modTime.IsZero() { + return o.modTime + } + + // Return the current time if can't read metadata + result = time.Now() + + // Read metadata (need ID) + err := o.readMetaData() + if err != nil { + fs.Debug(o, "Failed to read metadata: %v", err) + return result + } + + // Return the UploadTimestamp if can't get file info + result = time.Time(o.info.UploadTimestamp) + + // Now read the metadata for the modified time + opts := rest.Opts{ + Method: "POST", + Path: "/b2_get_file_info", + } + var request = api.GetFileInfoRequest{ + ID: o.info.ID, + } + var response api.FileInfo + _, err = o.fs.srv.CallJSON(&opts, &request, &response) + if err != nil { + fs.Debug(o, "Failed to get file info: %v", err) + return result + } + + // Parse the result + timeString := response.Info[timeKey] + parsed, err := parseTimeString(timeString) + if err != nil { + fs.Debug(o, "Failed to parse mod time string %q: %v", timeString, err) + return result + } + return parsed +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) { + // Not possible with B2 +} + +// Storable returns if this object is storable +func (o *Object) Storable() bool { + return true +} + +// openFile represents an Object open for reading +type openFile struct { + o *Object // Object we are reading for + resp *http.Response // response of the GET + body io.Reader // reading from here + hash hash.Hash // currently accumulating SHA1 + bytes int64 // number of bytes read on this connection + eof bool // whether we have read end of file +} + +// newOpenFile wraps an io.ReadCloser and checks the sha1sum +func newOpenFile(o *Object, resp *http.Response) *openFile { + file := &openFile{ + o: o, + resp: resp, + hash: sha1.New(), + } + file.body = io.TeeReader(resp.Body, file.hash) + return file +} + +// Read bytes from the object - see io.Reader +func (file *openFile) Read(p []byte) (n int, err error) { + n, err = file.body.Read(p) + file.bytes += int64(n) + if err == io.EOF { + file.eof = true + } + return +} + +// Close the object and checks the length and SHA1 if all the object +// was read +func (file *openFile) Close() (err error) { + // Close the body at the end + defer fs.CheckClose(file.resp.Body, &err) + + // If not end of file then can't check SHA1 + if !file.eof { + return nil + } + + // Check to see we read the correct number of bytes + if file.o.Size() != file.bytes { + return fmt.Errorf("Object corrupted on transfer - length mismatch (want %d got %d)", file.o.Size(), file.bytes) + } + + // Check the SHA1 + receivedSHA1 := file.resp.Header.Get(sha1Header) + calculatedSHA1 := fmt.Sprintf("%x", file.hash.Sum(nil)) + if receivedSHA1 != calculatedSHA1 { + return fmt.Errorf("Object corrupted on transfer - SHA1 mismatch (want %q got %q)", receivedSHA1, calculatedSHA1) + } + + return nil +} + +// Check it satisfies the interfaces +var _ io.ReadCloser = &openFile{} + +// Open an object for read +func (o *Object) Open() (in io.ReadCloser, err error) { + opts := rest.Opts{ + Method: "GET", + Absolute: true, + Path: o.fs.info.DownloadURL + "/file/" + urlEncode(o.fs.bucket) + "/" + urlEncode(o.fs.root+o.remote), + } + resp, err := o.fs.srv.Call(&opts) + if err != nil { + return nil, fmt.Errorf("Failed to open for download: %v", err) + } + + // Parse the time out of the headers if possible + timeString := resp.Header.Get(timeHeader) + parsed, err := parseTimeString(timeString) + if err != nil { + fs.Debug(o, "Failed to parse mod time string %q: %v", timeString, err) + } else { + o.modTime = parsed + } + return newOpenFile(o, resp), nil +} + +// dontEncode is the characters that do not need percent-encoding +// +// The characters that do not need percent-encoding are a subset of +// the printable ASCII characters: upper-case letters, lower-case +// letters, digits, ".", "_", "-", "/", "~", "!", "$", "'", "(", ")", +// "*", ";", "=", ":", and "@". All other byte values in a UTF-8 must +// be replaced with "%" and the two-digit hex value of the byte. +const dontEncode = (`abcdefghijklmnopqrstuvwxyz` + + `ABCDEFGHIJKLMNOPQRSTUVWXYZ` + + `0123456789` + + `._-/~!$'()*;=:@`) + +// noNeedToEncode is a bitmap of characters which don't need % encoding +var noNeedToEncode [256]bool + +func init() { + for _, c := range dontEncode { + noNeedToEncode[c] = true + } +} + +// urlEncode encodes in with % encoding +func urlEncode(in string) string { + var out bytes.Buffer + for i := 0; i < len(in); i++ { + c := in[i] + if noNeedToEncode[c] { + _ = out.WriteByte(c) + } else { + _, _ = out.WriteString(fmt.Sprintf("%%%2X", c)) + } + } + return out.String() +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, modTime time.Time, size int64) (err error) { + // Open a temp file to copy the input + fd, err := ioutil.TempFile("", "rclone-b2-") + if err != nil { + return err + } + _ = os.Remove(fd.Name()) // Delete the file - may not work on Windows + defer func() { + _ = fd.Close() // Ignore error may have been closed already + _ = os.Remove(fd.Name()) // Delete the file - may have been deleted already + }() + + // Copy the input while calculating the sha1 + hash := sha1.New() + teed := io.TeeReader(in, hash) + n, err := io.Copy(fd, teed) + if err != nil { + return err + } + if n != size { + return fmt.Errorf("Read %d bytes expecting %d", n, size) + } + calculatedSha1 := fmt.Sprintf("%x", hash.Sum(nil)) + + // Rewind the temporary file + _, err = fd.Seek(0, 0) + if err != nil { + return err + } + + // Get upload URL + UploadURL, AuthorizationToken, err := o.fs.getUploadURL() + if err != nil { + return err + } + + // Headers for upload file + // + // Authorization + // required + // An upload authorization token, from b2_get_upload_url. + // + // X-Bz-File-Name + // required + // + // The name of the file, in percent-encoded UTF-8. See Files for requirements on file names. See String Encoding. + // + // Content-Type + // required + // + // The MIME type of the content of the file, which will be returned in + // the Content-Type header when downloading the file. Use the + // Content-Type b2/x-auto to automatically set the stored Content-Type + // post upload. In the case where a file extension is absent or the + // lookup fails, the Content-Type is set to application/octet-stream. The + // Content-Type mappings can be purused here. + // + // X-Bz-Content-Sha1 + // required + // + // The SHA1 checksum of the content of the file. B2 will check this when + // the file is uploaded, to make sure that the file arrived correctly. It + // will be returned in the X-Bz-Content-Sha1 header when the file is + // downloaded. + // + // X-Bz-Info-src_last_modified_millis + // optional + // + // If the original source of the file being uploaded has a last modified + // time concept, Backblaze recommends using this spelling of one of your + // ten X-Bz-Info-* headers (see below). Using a standard spelling allows + // different B2 clients and the B2 web user interface to interoperate + // correctly. The value should be a base 10 number which represents a UTC + // time when the original source file was last modified. It is a base 10 + // number of milliseconds since midnight, January 1, 1970 UTC. This fits + // in a 64 bit integer such as the type "long" in the programming + // language Java. It is intended to be compatible with Java's time + // long. For example, it can be passed directly into the Java call + // Date.setTime(long time). + // + // X-Bz-Info-* + // optional + // + // Up to 10 of these headers may be present. The * part of the header + // name is replace with the name of a custom field in the file + // information stored with the file, and the value is an arbitrary UTF-8 + // string, percent-encoded. The same info headers sent with the upload + // will be returned with the download. + + opts := rest.Opts{ + Method: "POST", + Absolute: true, + Path: UploadURL, + Body: fd, + ExtraHeaders: map[string]string{ + "Authorization": AuthorizationToken, + "X-Bz-File-Name": urlEncode(o.fs.root + o.remote), + "Content-Type": fs.MimeType(o), + sha1Header: calculatedSha1, + timeHeader: timeString(modTime), + }, + ContentLength: &size, + } + var response api.FileInfo + _, err = o.fs.srv.CallJSON(&opts, nil, &response) + if err != nil { + return fmt.Errorf("Failed to upload: %v", err) + } + o.info.ID = response.ID + o.info.Name = response.Name + o.info.Action = "upload" + o.info.Size = response.Size + o.info.UploadTimestamp = api.Timestamp(time.Now()) // FIXME not quite right + return nil +} + +// Remove an object +func (o *Object) Remove() error { + bucketID, err := o.fs.getBucketID() + if err != nil { + return err + } + opts := rest.Opts{ + Method: "POST", + Path: "/b2_hide_file", + } + var request = api.HideFileRequest{ + BucketID: bucketID, + Name: o.info.Name, + } + var response api.File + _, err = o.fs.srv.CallJSON(&opts, &request, &response) + if err != nil { + return fmt.Errorf("Failed to delete file: %v", err) + } + return nil +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Purger = &Fs{} + _ fs.Object = &Object{} +) diff --git a/b2/b2_internal_test.go b/b2/b2_internal_test.go new file mode 100644 index 000000000..ef1ed851f --- /dev/null +++ b/b2/b2_internal_test.go @@ -0,0 +1,168 @@ +package b2 + +import ( + "testing" + "time" + + "github.com/ncw/rclone/fstest" +) + +// Test b2 string encoding +// https://www.backblaze.com/b2/docs/string_encoding.html + +var encodeTest = []struct { + fullyEncoded string + minimallyEncoded string + plainText string +}{ + {fullyEncoded: "%20", minimallyEncoded: "+", plainText: " "}, + {fullyEncoded: "%21", minimallyEncoded: "!", plainText: "!"}, + {fullyEncoded: "%22", minimallyEncoded: "%22", plainText: "\""}, + {fullyEncoded: "%23", minimallyEncoded: "%23", plainText: "#"}, + {fullyEncoded: "%24", minimallyEncoded: "$", plainText: "$"}, + {fullyEncoded: "%25", minimallyEncoded: "%25", plainText: "%"}, + {fullyEncoded: "%26", minimallyEncoded: "%26", plainText: "&"}, + {fullyEncoded: "%27", minimallyEncoded: "'", plainText: "'"}, + {fullyEncoded: "%28", minimallyEncoded: "(", plainText: "("}, + {fullyEncoded: "%29", minimallyEncoded: ")", plainText: ")"}, + {fullyEncoded: "%2A", minimallyEncoded: "*", plainText: "*"}, + {fullyEncoded: "%2B", minimallyEncoded: "%2B", plainText: "+"}, + {fullyEncoded: "%2C", minimallyEncoded: "%2C", plainText: ","}, + {fullyEncoded: "%2D", minimallyEncoded: "-", plainText: "-"}, + {fullyEncoded: "%2E", minimallyEncoded: ".", plainText: "."}, + {fullyEncoded: "%2F", minimallyEncoded: "/", plainText: "/"}, + {fullyEncoded: "%30", minimallyEncoded: "0", plainText: "0"}, + {fullyEncoded: "%31", minimallyEncoded: "1", plainText: "1"}, + {fullyEncoded: "%32", minimallyEncoded: "2", plainText: "2"}, + {fullyEncoded: "%33", minimallyEncoded: "3", plainText: "3"}, + {fullyEncoded: "%34", minimallyEncoded: "4", plainText: "4"}, + {fullyEncoded: "%35", minimallyEncoded: "5", plainText: "5"}, + {fullyEncoded: "%36", minimallyEncoded: "6", plainText: "6"}, + {fullyEncoded: "%37", minimallyEncoded: "7", plainText: "7"}, + {fullyEncoded: "%38", minimallyEncoded: "8", plainText: "8"}, + {fullyEncoded: "%39", minimallyEncoded: "9", plainText: "9"}, + {fullyEncoded: "%3A", minimallyEncoded: ":", plainText: ":"}, + {fullyEncoded: "%3B", minimallyEncoded: ";", plainText: ";"}, + {fullyEncoded: "%3C", minimallyEncoded: "%3C", plainText: "<"}, + {fullyEncoded: "%3D", minimallyEncoded: "=", plainText: "="}, + {fullyEncoded: "%3E", minimallyEncoded: "%3E", plainText: ">"}, + {fullyEncoded: "%3F", minimallyEncoded: "%3F", plainText: "?"}, + {fullyEncoded: "%40", minimallyEncoded: "@", plainText: "@"}, + {fullyEncoded: "%41", minimallyEncoded: "A", plainText: "A"}, + {fullyEncoded: "%42", minimallyEncoded: "B", plainText: "B"}, + {fullyEncoded: "%43", minimallyEncoded: "C", plainText: "C"}, + {fullyEncoded: "%44", minimallyEncoded: "D", plainText: "D"}, + {fullyEncoded: "%45", minimallyEncoded: "E", plainText: "E"}, + {fullyEncoded: "%46", minimallyEncoded: "F", plainText: "F"}, + {fullyEncoded: "%47", minimallyEncoded: "G", plainText: "G"}, + {fullyEncoded: "%48", minimallyEncoded: "H", plainText: "H"}, + {fullyEncoded: "%49", minimallyEncoded: "I", plainText: "I"}, + {fullyEncoded: "%4A", minimallyEncoded: "J", plainText: "J"}, + {fullyEncoded: "%4B", minimallyEncoded: "K", plainText: "K"}, + {fullyEncoded: "%4C", minimallyEncoded: "L", plainText: "L"}, + {fullyEncoded: "%4D", minimallyEncoded: "M", plainText: "M"}, + {fullyEncoded: "%4E", minimallyEncoded: "N", plainText: "N"}, + {fullyEncoded: "%4F", minimallyEncoded: "O", plainText: "O"}, + {fullyEncoded: "%50", minimallyEncoded: "P", plainText: "P"}, + {fullyEncoded: "%51", minimallyEncoded: "Q", plainText: "Q"}, + {fullyEncoded: "%52", minimallyEncoded: "R", plainText: "R"}, + {fullyEncoded: "%53", minimallyEncoded: "S", plainText: "S"}, + {fullyEncoded: "%54", minimallyEncoded: "T", plainText: "T"}, + {fullyEncoded: "%55", minimallyEncoded: "U", plainText: "U"}, + {fullyEncoded: "%56", minimallyEncoded: "V", plainText: "V"}, + {fullyEncoded: "%57", minimallyEncoded: "W", plainText: "W"}, + {fullyEncoded: "%58", minimallyEncoded: "X", plainText: "X"}, + {fullyEncoded: "%59", minimallyEncoded: "Y", plainText: "Y"}, + {fullyEncoded: "%5A", minimallyEncoded: "Z", plainText: "Z"}, + {fullyEncoded: "%5B", minimallyEncoded: "%5B", plainText: "["}, + {fullyEncoded: "%5C", minimallyEncoded: "%5C", plainText: "\\"}, + {fullyEncoded: "%5D", minimallyEncoded: "%5D", plainText: "]"}, + {fullyEncoded: "%5E", minimallyEncoded: "%5E", plainText: "^"}, + {fullyEncoded: "%5F", minimallyEncoded: "_", plainText: "_"}, + {fullyEncoded: "%60", minimallyEncoded: "%60", plainText: "`"}, + {fullyEncoded: "%61", minimallyEncoded: "a", plainText: "a"}, + {fullyEncoded: "%62", minimallyEncoded: "b", plainText: "b"}, + {fullyEncoded: "%63", minimallyEncoded: "c", plainText: "c"}, + {fullyEncoded: "%64", minimallyEncoded: "d", plainText: "d"}, + {fullyEncoded: "%65", minimallyEncoded: "e", plainText: "e"}, + {fullyEncoded: "%66", minimallyEncoded: "f", plainText: "f"}, + {fullyEncoded: "%67", minimallyEncoded: "g", plainText: "g"}, + {fullyEncoded: "%68", minimallyEncoded: "h", plainText: "h"}, + {fullyEncoded: "%69", minimallyEncoded: "i", plainText: "i"}, + {fullyEncoded: "%6A", minimallyEncoded: "j", plainText: "j"}, + {fullyEncoded: "%6B", minimallyEncoded: "k", plainText: "k"}, + {fullyEncoded: "%6C", minimallyEncoded: "l", plainText: "l"}, + {fullyEncoded: "%6D", minimallyEncoded: "m", plainText: "m"}, + {fullyEncoded: "%6E", minimallyEncoded: "n", plainText: "n"}, + {fullyEncoded: "%6F", minimallyEncoded: "o", plainText: "o"}, + {fullyEncoded: "%70", minimallyEncoded: "p", plainText: "p"}, + {fullyEncoded: "%71", minimallyEncoded: "q", plainText: "q"}, + {fullyEncoded: "%72", minimallyEncoded: "r", plainText: "r"}, + {fullyEncoded: "%73", minimallyEncoded: "s", plainText: "s"}, + {fullyEncoded: "%74", minimallyEncoded: "t", plainText: "t"}, + {fullyEncoded: "%75", minimallyEncoded: "u", plainText: "u"}, + {fullyEncoded: "%76", minimallyEncoded: "v", plainText: "v"}, + {fullyEncoded: "%77", minimallyEncoded: "w", plainText: "w"}, + {fullyEncoded: "%78", minimallyEncoded: "x", plainText: "x"}, + {fullyEncoded: "%79", minimallyEncoded: "y", plainText: "y"}, + {fullyEncoded: "%7A", minimallyEncoded: "z", plainText: "z"}, + {fullyEncoded: "%7B", minimallyEncoded: "%7B", plainText: "{"}, + {fullyEncoded: "%7C", minimallyEncoded: "%7C", plainText: "|"}, + {fullyEncoded: "%7D", minimallyEncoded: "%7D", plainText: "}"}, + {fullyEncoded: "%7E", minimallyEncoded: "~", plainText: "~"}, + {fullyEncoded: "%7F", minimallyEncoded: "%7F", plainText: "\u007f"}, + {fullyEncoded: "%E8%87%AA%E7%94%B1", minimallyEncoded: "%E8%87%AA%E7%94%B1", plainText: "自由"}, + {fullyEncoded: "%F0%90%90%80", minimallyEncoded: "%F0%90%90%80", plainText: "𐐀"}, +} + +func TestUrlEncode(t *testing.T) { + for _, test := range encodeTest { + got := urlEncode(test.plainText) + if got != test.minimallyEncoded && got != test.fullyEncoded { + t.Errorf("urlEncode(%q) got %q wanted %q or %q", test.plainText, got, test.minimallyEncoded, test.fullyEncoded) + } + } +} + +func TestTimeString(t *testing.T) { + for _, test := range []struct { + in time.Time + want string + }{ + {fstest.Time("1970-01-01T00:00:00.000000000Z"), "0"}, + {fstest.Time("2001-02-03T04:05:10.123123123Z"), "981173110123"}, + {fstest.Time("2001-02-03T05:05:10.123123123+01:00"), "981173110123"}, + } { + got := timeString(test.in) + if test.want != got { + t.Logf("%v: want %v got %v", test.in, test.want, got) + } + } + +} + +func TestParseTimeString(t *testing.T) { + for _, test := range []struct { + in string + want time.Time + wantError string + }{ + {"0", fstest.Time("1970-01-01T00:00:00.000000000Z"), ""}, + {"981173110123", fstest.Time("2001-02-03T04:05:10.123000000Z"), ""}, + {"", time.Time{}, `"src_last_modified_millis" not found in metadata`}, + {"potato", time.Time{}, `strconv.ParseInt: parsing "potato": invalid syntax`}, + } { + got, err := parseTimeString(test.in) + var gotError string + if err != nil { + gotError = err.Error() + } + if test.want != got { + t.Logf("%v: want %v got %v", test.in, test.want, got) + } + if test.wantError != gotError { + t.Logf("%v: want error %v got error %v", test.in, test.wantError, gotError) + } + } + +} diff --git a/b2/b2_test.go b/b2/b2_test.go new file mode 100644 index 000000000..7d09b35e5 --- /dev/null +++ b/b2/b2_test.go @@ -0,0 +1,56 @@ +// Test B2 filesystem interface +// +// Automatically generated - DO NOT EDIT +// Regenerate with: make gen_tests +package b2_test + +import ( + "testing" + + "github.com/ncw/rclone/b2" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" +) + +func init() { + fstests.NilObject = fs.Object((*b2.Object)(nil)) + fstests.RemoteName = "TestB2:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/docs/content/about.md b/docs/content/about.md index f0b93a1e8..89b8cc0aa 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -21,6 +21,7 @@ Rclone is a command line program to sync files and directories to and from * Amazon Cloud Drive * Microsoft One Drive * Hubic + * Backblaze B2 * The local filesystem Features diff --git a/docs/content/b2.md b/docs/content/b2.md new file mode 100644 index 000000000..948f34771 --- /dev/null +++ b/docs/content/b2.md @@ -0,0 +1,124 @@ +--- +title: "B2" +description: "Backblaze B2" +date: "2015-12-29" +--- + +Backblaze B2 +---------------------------------------- + +B2 is [Backblaze's cloud storage system](https://www.backblaze.com/b2/). + +Paths are specified as `remote:bucket` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, eg `remote:bucket/path/to/dir`. + +Here is an example of making a b2 configuration. First run + + rclone config + +This will guide you through an interactive setup process. You will +need your account number (a short hex number) and key (a long hex +number) which you can get from the b2 control panel. + +``` +No remotes found - make a new one +n) New remote +q) Quit config +n/q> n +name> remote +What type of source is it? +Choose a number from below + 1) amazon cloud drive + 2) b2 + 3) drive + 4) dropbox + 5) google cloud storage + 6) swift + 7) hubic + 8) local + 9) onedrive +10) s3 +type> 2 +Account ID +account> 123456789abc +Application Key +key> 0123456789abcdef0123456789abcdef0123456789 +Endpoint for the service - leave blank normally. +endpoint> +Remote config +-------------------- +[remote] +account = 123456789abc +key = 0123456789abcdef0123456789abcdef0123456789 +endpoint = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This remote is called `remote` and can now be used like this + +See all buckets + + rclone lsd remote: + +Make a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + +Sync `/home/local/directory` to the remote bucket, deleting any +excess files in the bucket. + + rclone sync /home/local/directory remote:bucket + +### Modified time ### + +The modified time is stored as metadata on the object as +`X-Bz-Info-src_last_modified_millis` as milliseconds since 1970-01-01 +in the Backblaze standard. Other tools should be able to use this as +a modified time. + +Modified times are set on upload, read on download and shown in +listings. They are not used in syncing as unfortunately B2 doesn't +have an API method to set them independently of doing an upload. + +### SHA1 checksums ### + +The SHA1 checksums of the files are checked on upload and download, +but they aren't used for sync as they aren't compatible with the MD5 +checksums rclone normally uses. + +### Versions ### + +When rclone uploads a new version of a file it creates a [new version +of it](https://www.backblaze.com/b2/docs/file_versions.html). +Likewise when you delete a file, the old version will still be +available. + +The old versions of files are visible in the B2 web interface, but not +via rclone yet. + +Rclone doesn't provide any way of managing old versions (downloading +them or deleting them) at the moment. When you `purge` a bucket, all +the old versions will be deleted. + +### Bugs ### + +Note that when uploading a file, rclone has to make a temporary copy +of it in your temp filing system. This is due to a weakness in the B2 +API which I'm hoping will be addressed soon. + +### API ### + +Here are [some notes I made on the backblaze +API](https://gist.github.com/ncw/166dabf352b399f1cc1c) while +integrating it with rclone which detail the changes I'd like to see. +With a couple of small tweaks Backblaze could enable rclone to not +make a temporary copy of all files and fully support modification +times. diff --git a/docs/content/overview.md b/docs/content/overview.md index 85dad9004..910904d78 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -25,6 +25,7 @@ Here is an overview of the major features of each cloud storage system. | Amazon Cloud Drive | Yes | No | Yes | No | | Microsoft One Drive | No | Yes | Yes | No | | Hubic | Yes | Yes | No | No | +| Backblaze B2 | No | Partial | No | No | | The local filesystem | Yes | Yes | Depends | No | ### MD5SUM ### @@ -44,6 +45,9 @@ default, though the MD5SUM can be checked with the `--checksum` flag. All cloud storage systems support some kind of date on the object and these will be set when transferring from the cloud storage system. +Backblaze B2 preserves file modification times on files uploaded and +downloaded, but doesn't use them to decide which objects to sync. + ### Case Insensitive ### If a cloud storage systems is case sensitive then it is possible to diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 0fb445690..acf5de203 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -39,6 +39,7 @@
  • Amazon Cloud Drive
  • Microsoft One Drive
  • Hubic
    • +
  • Backblaze B2
  • Local
    • diff --git a/fs/operations_test.go b/fs/operations_test.go index e45f27095..ea8c03b25 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -21,6 +21,7 @@ import ( // Active file systems _ "github.com/ncw/rclone/amazonclouddrive" + _ "github.com/ncw/rclone/b2" _ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/googlecloudstorage" diff --git a/fs/test_all.go b/fs/test_all.go index a80630d04..43ebb64e2 100644 --- a/fs/test_all.go +++ b/fs/test_all.go @@ -25,6 +25,7 @@ var ( "TestAmazonCloudDrive:", "TestOneDrive:", "TestHubic:", + "TestB2:", } binary = "fs.test" // Flags diff --git a/fstest/fstests/gen_tests.go b/fstest/fstests/gen_tests.go index 52b8871aa..b5732bb7a 100644 --- a/fstest/fstests/gen_tests.go +++ b/fstest/fstests/gen_tests.go @@ -133,5 +133,6 @@ func main() { generateTestProgram(t, fns, "AmazonCloudDrive") generateTestProgram(t, fns, "OneDrive") generateTestProgram(t, fns, "Hubic") + generateTestProgram(t, fns, "B2") log.Printf("Done") } diff --git a/make_manual.py b/make_manual.py index e14309b69..86f99e7bf 100755 --- a/make_manual.py +++ b/make_manual.py @@ -26,6 +26,7 @@ docs = [ "amazonclouddrive.md", "onedrive.md", "hubic.md", + "b2.md", "local.md", "changelog.md", "bugs.md", diff --git a/onedrive/onedrive.go b/onedrive/onedrive.go index cd4088b47..ceaa5025f 100644 --- a/onedrive/onedrive.go +++ b/onedrive/onedrive.go @@ -177,7 +177,7 @@ func NewFs(name, root string) (fs.Fs, error) { f := &Fs{ name: name, root: root, - srv: rest.NewClient(oAuthClient, rootURL), + srv: rest.NewClient(oAuthClient).SetRoot(rootURL), pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), } f.srv.SetErrorHandler(errorHandler) diff --git a/rclone.go b/rclone.go index 878ddfb50..f6c201211 100644 --- a/rclone.go +++ b/rclone.go @@ -17,6 +17,7 @@ import ( "github.com/ncw/rclone/fs" // Active file systems _ "github.com/ncw/rclone/amazonclouddrive" + _ "github.com/ncw/rclone/b2" _ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/googlecloudstorage" diff --git a/rest/rest.go b/rest/rest.go index 65c17d80e..20ac23a3c 100644 --- a/rest/rest.go +++ b/rest/rest.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "net/http" "github.com/ncw/rclone/fs" @@ -16,27 +17,48 @@ type Client struct { c *http.Client rootURL string errorHandler func(resp *http.Response) error + headers map[string]string } // NewClient takes an oauth http.Client and makes a new api instance -func NewClient(c *http.Client, rootURL string) *Client { - return &Client{ +func NewClient(c *http.Client) *Client { + api := &Client{ c: c, - rootURL: rootURL, errorHandler: defaultErrorHandler, + headers: make(map[string]string), } + api.SetHeader("User-Agent", fs.UserAgent) + return api } -// defaultErrorHandler doesn't attempt to parse the http body +// defaultErrorHandler doesn't attempt to parse the http body, just +// returns it in the error message func defaultErrorHandler(resp *http.Response) (err error) { defer fs.CheckClose(resp.Body, &err) - return fmt.Errorf("HTTP error %v (%v) returned", resp.StatusCode, resp.Status) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + return fmt.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body) } // SetErrorHandler sets the handler to decode an error response when // the HTTP status code is not 2xx. The handler should close resp.Body. -func (api *Client) SetErrorHandler(fn func(resp *http.Response) error) { +func (api *Client) SetErrorHandler(fn func(resp *http.Response) error) *Client { api.errorHandler = fn + return api +} + +// SetRoot sets the default root URL +func (api *Client) SetRoot(RootURL string) *Client { + api.rootURL = RootURL + return api +} + +// SetHeader sets a header for all requests +func (api *Client) SetHeader(key, value string) *Client { + api.headers[key] = value + return api } // Opts contains parameters for Call, CallJSON etc @@ -50,6 +72,8 @@ type Opts struct { ContentLength *int64 ContentRange string ExtraHeaders map[string]string + UserName string // username for Basic Auth + Password string // password for Basic Auth } // DecodeJSON decodes resp.Body into result @@ -72,27 +96,42 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) { if opts.Absolute { url = opts.Path } else { + if api.rootURL == "" { + return nil, fmt.Errorf("RootURL not set") + } url = api.rootURL + opts.Path } req, err := http.NewRequest(opts.Method, url, opts.Body) if err != nil { return } + headers := make(map[string]string) + // Set default headers + for k, v := range api.headers { + headers[k] = v + } if opts.ContentType != "" { - req.Header.Add("Content-Type", opts.ContentType) + headers["Content-Type"] = opts.ContentType } if opts.ContentLength != nil { req.ContentLength = *opts.ContentLength } if opts.ContentRange != "" { - req.Header.Add("Content-Range", opts.ContentRange) + headers["Content-Range"] = opts.ContentRange } + // Set any extra headers if opts.ExtraHeaders != nil { for k, v := range opts.ExtraHeaders { - req.Header.Add(k, v) + headers[k] = v } } - req.Header.Add("User-Agent", fs.UserAgent) + // Now set the headers + for k, v := range headers { + req.Header.Add(k, v) + } + if opts.UserName != "" || opts.Password != "" { + req.SetBasicAuth(opts.UserName, opts.Password) + } resp, err = api.c.Do(req) if err != nil { return nil, err @@ -127,7 +166,7 @@ func (api *Client) CallJSON(opts *Opts, request interface{}, response interface{ if err != nil { return resp, err } - if opts.NoResponse { + if response == nil || opts.NoResponse { return resp, nil } err = DecodeJSON(resp, response)