TrueCloudLab/rclone
16
0
Fork 4
Code Issues 3 Pull requests 2 Projects Releases 3 Packages Wiki Activity Actions

Implement Backblaze B2 - fixes #224

Browse source
This commit is contained in:
Nick Craig-Wood 2015-11-27 18:25:52 +00:00
parent 113624691a
commit 33b3eea6ec
16 changed files with 1530 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -19,6 +19,7 @@ Rclone is a command line program to sync files and directories to and from
* Amazon Cloud Drive * Amazon Cloud Drive
* Microsoft One Drive * Microsoft One Drive
* Hubic * Hubic
* Backblaze B2
* The local filesystem * The local filesystem
Features Features

147
b2/api/types.go Normal file
View file

@ -0,0 +1,147 @@
package api
import (
"fmt"
"strconv"
"time"
)
// Error describes a B2 error response
type Error struct {
Status int `json:"status"` // The numeric HTTP status code. Always matches the status in the HTTP response.
Code string `json:"code"` // A single-identifier code that identifies the error.
Message string `json:"message"` // A human-readable message, in English, saying what went wrong.
}
// Error statisfies the error interface
func (e *Error) Error() string {
return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code)
}
// Account describes a B2 account
type Account struct {
ID string `json:"accountId"` // The identifier for the account.
}
// Bucket describes a B2 bucket
type Bucket struct {
ID string `json:"bucketId"`
AccountID string `json:"accountId"`
Name string `json:"bucketName"`
Type string `json:"bucketType"`
}
// Timestamp is a UTC time when this file was uploaded. It is a base
// 10 number of milliseconds since midnight, January 1, 1970 UTC. This
// fits in a 64 bit integer such as the type "long" in the programming
// language Java. It is intended to be compatible with Java's time
// long. For example, it can be passed directly into the java call
// Date.setTime(long time).
type Timestamp time.Time
// MarshalJSON turns a Timestamp into JSON (in UTC)
func (t *Timestamp) MarshalJSON() (out []byte, err error) {
timestamp := (*time.Time)(t).UTC().UnixNano()
return []byte(strconv.FormatInt(timestamp/1E6, 10)), nil
}
// UnmarshalJSON turns JSON into a Timestamp
func (t *Timestamp) UnmarshalJSON(data []byte) error {
timestamp, err := strconv.ParseInt(string(data), 10, 64)
if err != nil {
return err
}
*t = Timestamp(time.Unix(timestamp/1E3, (timestamp%1E3)*1E6))
return nil
}
// File is info about a file
type File struct {
ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version.
Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name.
Action string `json:"action"` // Either "upload" or "hide". "upload" means a file that was uploaded to B2 Cloud Storage. "hide" means a file version marking the file as hidden, so that it will not show up in b2_list_file_names. The result of b2_list_file_names will contain only "upload". The result of b2_list_file_versions may have both.
Size int64 `json:"size"` // The number of bytes in the file.
UploadTimestamp Timestamp `json:"uploadTimestamp"` // This is a UTC time when this file was uploaded.
}
// AuthorizeAccountResponse is as returned from the b2_authorize_account call
type AuthorizeAccountResponse struct {
AccountID string `json:"accountId"` // The identifier for the account.
AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.
APIURL string `json:"apiUrl"` // The base URL to use for all API calls except for uploading and downloading files.
DownloadURL string `json:"downloadUrl"` // The base URL to use for downloading files.
}
// ListBucketsResponse is as returned from the b2_list_buckets call
type ListBucketsResponse struct {
Buckets []Bucket `json:"buckets"`
}
// ListFileNamesRequest is as passed to b2_list_file_names or b2_list_file_versions
type ListFileNamesRequest struct {
BucketID string `json:"bucketId"` // required - The bucket to look for file names in.
StartFileName string `json:"startFileName,omitempty"` // optional - The first file name to return. If there is a file with this name, it will be returned in the list. If not, the first file name after this the first one after this name.
MaxFileCount int `json:"maxFileCount,omitempty"` // optional - The maximum number of files to return from this call. The default value is 100, and the maximum allowed is 1000.
StartFileID string `json:"startFileId,omitempty"` // optional - What to pass in to startFileId for the next search to continue where this one left off.
}
// ListFileNamesResponse is as received from b2_list_file_names or b2_list_file_versions
type ListFileNamesResponse struct {
Files []File `json:"files"` // An array of objects, each one describing one file.
NextFileName *string `json:"nextFileName"` // What to pass in to startFileName for the next search to continue where this one left off, or null if there are no more files.
NextFileID *string `json:"nextFileId"` // What to pass in to startFileId for the next search to continue where this one left off, or null if there are no more files.
}
// GetUploadURLRequest is passed to b2_get_upload_url
type GetUploadURLRequest struct {
BucketID string `json:"bucketId"` // The ID of the bucket that you want to upload to.
}
// GetUploadURLResponse is received from b2_get_upload_url
type GetUploadURLResponse struct {
BucketID string `json:"bucketId"` // The unique ID of the bucket.
UploadURL string `json:"uploadUrl"` // The URL that can be used to upload files to this bucket, see b2_upload_file.
AuthorizationToken string `json:"authorizationToken"` // The authorizationToken that must be used when uploading files to this bucket, see b2_upload_file.
}
// FileInfo is received from b2_upload_file and b2_get_file_info
type FileInfo struct {
ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version.
Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name.
AccountID string `json:"accountId"` // Your account ID.
BucketID string `json:"bucketId"` // The bucket that the file is in.
Size int64 `json:"contentLength"` // The number of bytes stored in the file.
SHA1 string `json:"contentSha1"` // The SHA1 of the bytes stored in the file.
ContentType string `json:"contentType"` // The MIME type of the file.
Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file.
}
// CreateBucketRequest is used to create a bucket
type CreateBucketRequest struct {
AccountID string `json:"accountId"`
Name string `json:"bucketName"`
Type string `json:"bucketType"`
}
// DeleteBucketRequest is used to create a bucket
type DeleteBucketRequest struct {
ID string `json:"bucketId"`
AccountID string `json:"accountId"`
}
// DeleteFileRequest is used to delete a file version
type DeleteFileRequest struct {
ID string `json:"fileId"` // The ID of the file, as returned by b2_upload_file, b2_list_file_names, or b2_list_file_versions.
Name string `json:"fileName"` // The name of this file.
}
// HideFileRequest is used to delete a file
type HideFileRequest struct {
BucketID string `json:"bucketId"` // The bucket containing the file to hide.
Name string `json:"fileName"` // The name of the file to hide.
}
// GetFileInfoRequest is used to return a FileInfo struct with b2_get_file_info
type GetFileInfoRequest struct {
ID string `json:"fileId"` // The ID of the file, as returned by b2_upload_file, b2_list_file_names, or b2_list_file_versions.
}

972
b2/b2.go Normal file
View file

@ -0,0 +1,972 @@
// Package b2 provides an interface to the Backblaze B2 object storage system
package b2
// FIXME if b2 could set the mod time then it has everything else to
// implement mod times. It is just missing that bit of API.
import (
"bytes"
"crypto/sha1"
"errors"
"fmt"
"hash"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/ncw/rclone/b2/api"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/rest"
)
const (
defaultEndpoint = "https://api.backblaze.com"
headerPrefix = "x-bz-info-" // lower case as that is what the server returns
timeKey = "src_last_modified_millis"
timeHeader = headerPrefix + timeKey
sha1Header = "X-Bz-Content-Sha1"
)
// Register with Fs
func init() {
fs.Register(&fs.Info{
Name: "b2",
NewFs: NewFs,
Options: []fs.Option{{
Name: "account",
Help: "Account ID",
}, {
Name: "key",
Help: "Application Key",
}, {
Name: "endpoint",
Help: "Endpoint for the service - leave blank normally.",
},
},
})
}
// Fs represents a remote b2 server
type Fs struct {
name string // name of this remote
srv *rest.Client // the connection to the b2 server
bucket string // the bucket we are working on
bucketIDMutex sync.Mutex // mutex to protect _bucketID
_bucketID string // the ID of the bucket we are working on
root string // the path we are working on if any
info api.AuthorizeAccountResponse // result of authorize call
uploadMu sync.Mutex // lock for upload variable
upload api.GetUploadURLResponse // result of get upload URL call
}
// Object describes a b2 object
//
// Will definitely have info
type Object struct {
fs *Fs // what this object is part of
remote string // The remote path
info api.File // Info from the b2 object if known
modTime time.Time // The modified time of the object if known
}
// ------------------------------------------------------------
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
}
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
if f.root == "" {
return f.bucket
}
return f.bucket + "/" + f.root
}
// String converts this Fs to a string
func (f *Fs) String() string {
if f.root == "" {
return fmt.Sprintf("B2 bucket %s", f.bucket)
}
return fmt.Sprintf("B2 bucket %s path %s", f.bucket, f.root)
}
// Pattern to match a b2 path
var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
// parseParse parses a b2 'url'
func parsePath(path string) (bucket, directory string, err error) {
parts := matcher.FindStringSubmatch(path)
if parts == nil {
err = fmt.Errorf("Couldn't find bucket in b2 path %q", path)
} else {
bucket, directory = parts[1], parts[2]
directory = strings.Trim(directory, "/")
}
return
}
// errorHandler parses a non 2xx error response into an error
func errorHandler(resp *http.Response) error {
// Decode error response
errResponse := new(api.Error)
err := rest.DecodeJSON(resp, &errResponse)
if err != nil {
fs.Debug(nil, "Couldn't decode error response: %v", err)
}
if errResponse.Code == "" {
errResponse.Code = "unknown"
}
if errResponse.Status == 0 {
errResponse.Status = resp.StatusCode
}
if errResponse.Message == "" {
errResponse.Message = "Unknown " + resp.Status
}
return errResponse
}
// NewFs contstructs an Fs from the path, bucket:path
func NewFs(name, root string) (fs.Fs, error) {
bucket, directory, err := parsePath(root)
if err != nil {
return nil, err
}
f := &Fs{
name: name,
bucket: bucket,
root: directory,
}
account := fs.ConfigFile.MustValue(name, "account")
if account == "" {
return nil, errors.New("account not found")
}
key := fs.ConfigFile.MustValue(name, "key")
if key == "" {
return nil, errors.New("key not found")
}
endpoint := fs.ConfigFile.MustValue(name, "endpoint", defaultEndpoint)
f.srv = rest.NewClient(fs.Config.Client()).SetRoot(endpoint + "/b2api/v1").SetErrorHandler(errorHandler)
opts := rest.Opts{
Method: "GET",
Path: "/b2_authorize_account",
UserName: account,
Password: key,
}
_, err = f.srv.CallJSON(&opts, nil, &f.info)
if err != nil {
return nil, fmt.Errorf("Failed to authenticate: %v", err)
}
f.srv.SetRoot(f.info.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken)
if f.root != "" {
f.root += "/"
// Check to see if the (bucket,directory) is actually an existing file
oldRoot := f.root
remote := path.Base(directory)
f.root = path.Dir(directory)
if f.root == "." {
f.root = ""
} else {
f.root += "/"
}
obj := f.NewFsObject(remote)
if obj != nil {
return fs.NewLimited(f, obj), nil
}
f.root = oldRoot
}
return f, nil
}
// getUploadURL returns the UploadURL and the AuthorizationToken
func (f *Fs) getUploadURL() (string, string, error) {
f.uploadMu.Lock()
defer f.uploadMu.Unlock()
bucketID, err := f.getBucketID()
if err != nil {
return "", "", err
}
if f.upload.UploadURL == "" || f.upload.AuthorizationToken == "" {
opts := rest.Opts{
Method: "POST",
Path: "/b2_get_upload_url",
}
var request = api.GetUploadURLRequest{
BucketID: bucketID,
}
_, err := f.srv.CallJSON(&opts, &request, &f.upload)
if err != nil {
return "", "", fmt.Errorf("Failed to get upload URL: %v", err)
}
}
return f.upload.UploadURL, f.upload.AuthorizationToken, nil
}
// clearUploadURL clears the current UploadURL and the AuthorizationToken
func (f *Fs) clearUploadURL() {
f.uploadMu.Lock()
f.upload = api.GetUploadURLResponse{}
defer f.uploadMu.Unlock()
}
// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *Fs) newFsObjectWithInfo(remote string, info *api.File) fs.Object {
o := &Object{
fs: f,
remote: remote,
}
if info != nil {
// Set info but not headers
o.info = *info
} else {
err := o.readMetaData() // reads info and headers, returning an error
if err != nil {
fs.Debug(o, "Failed to read metadata: %s", err)
return nil
}
}
return o
}
// NewFsObject returns an FsObject from a path
//
// May return nil if an error occurred
func (f *Fs) NewFsObject(remote string) fs.Object {
return f.newFsObjectWithInfo(remote, nil)
}
// listFn is called from list to handle an object
type listFn func(string, *api.File) error
// list lists the objects into the function supplied from
// the bucket and root supplied
//
// If prefix is set then startFileName is used as a prefix which all
// files must have
//
// If limit is > 0 then it limits to that many files (must be less
// than 1000)
//
// If hidden is set then it will list the hidden (deleted) files too.
func (f *Fs) list(prefix string, limit int, hidden bool, fn listFn) error {
bucketID, err := f.getBucketID()
if err != nil {
return err
}
chunkSize := 1000
if limit > 0 {
chunkSize = limit
}
var request = api.ListFileNamesRequest{
BucketID: bucketID,
MaxFileCount: chunkSize,
}
prefix = f.root + prefix
if prefix != "" {
request.StartFileName = prefix
}
var response api.ListFileNamesResponse
opts := rest.Opts{
Method: "POST",
Path: "/b2_list_file_names",
}
if hidden {
opts.Path = "/b2_list_file_versions"
}
for {
_, err = f.srv.CallJSON(&opts, &request, &response)
if err != nil {
return err
}
for i := range response.Files {
file := &response.Files[i]
// Finish if file name no longer has prefix
if !strings.HasPrefix(file.Name, prefix) {
return nil
}
err = fn(file.Name[len(prefix):], file)
if err != nil {
return err
}
}
// end if no NextFileName
if response.NextFileName == nil {
break
}
request.StartFileName = *response.NextFileName
if response.NextFileID != nil {
request.StartFileID = *response.NextFileID
}
}
return nil
}
// List walks the path returning a channel of FsObjects
func (f *Fs) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers)
if f.bucket == "" {
// Return no objects at top level list
close(out)
fs.Stats.Error()
fs.ErrorLog(f, "Can't list objects at root - choose a bucket using lsd")
} else {
// List the objects
go func() {
defer close(out)
err := f.list("", 0, false, func(remote string, object *api.File) error {
if o := f.newFsObjectWithInfo(remote, object); o != nil {
out <- o
}
return nil
})
if err != nil {
fs.Stats.Error()
fs.ErrorLog(f, "Couldn't list bucket %q: %s", f.bucket, err)
}
}()
}
return out
}
// listBucketFn is called from listBuckets to handle a bucket
type listBucketFn func(*api.Bucket)
// listBuckets lists the buckets to the function supplied
func (f *Fs) listBuckets(fn listBucketFn) error {
var account = api.Account{ID: f.info.AccountID}
var response api.ListBucketsResponse
opts := rest.Opts{
Method: "POST",
Path: "/b2_list_buckets",
}
_, err := f.srv.CallJSON(&opts, &account, &response)
if err != nil {
return err
}
for i := range response.Buckets {
fn(&response.Buckets[i])
}
return nil
}
// getBucketID finds the ID for the current bucket name
func (f *Fs) getBucketID() (bucketID string, err error) {
f.bucketIDMutex.Lock()
defer f.bucketIDMutex.Unlock()
if f._bucketID != "" {
return f._bucketID, nil
}
err = f.listBuckets(func(bucket *api.Bucket) {
if bucket.Name == f.bucket {
bucketID = bucket.ID
}
})
if bucketID == "" {
err = fmt.Errorf("Couldn't find bucket %q", f.bucket)
}
f._bucketID = bucketID
return bucketID, err
}
// setBucketID sets the ID for the current bucket name
func (f *Fs) setBucketID(ID string) {
f.bucketIDMutex.Lock()
f._bucketID = ID
f.bucketIDMutex.Unlock()
}
// clearBucketID clears the ID for the current bucket name
func (f *Fs) clearBucketID() {
f.bucketIDMutex.Lock()
f._bucketID = ""
f.bucketIDMutex.Unlock()
}
// ListDir lists the buckets
func (f *Fs) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers)
if f.bucket == "" {
// List the buckets
go func() {
defer close(out)
err := f.listBuckets(func(bucket *api.Bucket) {
out <- &fs.Dir{
Name: bucket.Name,
Bytes: -1,
Count: -1,
}
})
if err != nil {
fs.Stats.Error()
fs.ErrorLog(f, "Error listing buckets: %v", err)
}
}()
} else {
// List the directories in the path in the bucket
go func() {
defer close(out)
lastDir := ""
err := f.list("", 0, false, func(remote string, object *api.File) error {
slash := strings.IndexRune(remote, '/')
if slash < 0 {
return nil
}
dir := remote[:slash]
if dir == lastDir {
return nil
}
out <- &fs.Dir{
Name: dir,
Bytes: -1,
Count: -1,
}
lastDir = dir
return nil
})
if err != nil {
fs.Stats.Error()
fs.ErrorLog(f, "Couldn't list bucket %q: %s", f.bucket, err)
}
}()
}
return out
}
// Put the object into the bucket
//
// Copy the reader in to the new object which is returned
//
// The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary Object under construction
fs := &Object{
fs: f,
remote: remote,
}
return fs, fs.Update(in, modTime, size)
}
// Mkdir creates the bucket if it doesn't exist
func (f *Fs) Mkdir() error {
opts := rest.Opts{
Method: "POST",
Path: "/b2_create_bucket",
}
var request = api.CreateBucketRequest{
AccountID: f.info.AccountID,
Name: f.bucket,
Type: "allPrivate",
}
var response api.Bucket
_, err := f.srv.CallJSON(&opts, &request, &response)
if err != nil {
if apiErr, ok := err.(*api.Error); ok {
if apiErr.Code == "duplicate_bucket_name" {
return nil
}
}
return fmt.Errorf("Failed to create bucket: %v", err)
}
f.setBucketID(response.ID)
return nil
}
// Rmdir deletes the bucket if the fs is at the root
//
// Returns an error if it isn't empty
func (f *Fs) Rmdir() error {
if f.root != "" {
return nil
}
opts := rest.Opts{
Method: "POST",
Path: "/b2_delete_bucket",
}
bucketID, err := f.getBucketID()
if err != nil {
return err
}
var request = api.DeleteBucketRequest{
ID: bucketID,
AccountID: f.info.AccountID,
}
var response api.Bucket
_, err = f.srv.CallJSON(&opts, &request, &response)
if err != nil {
return fmt.Errorf("Failed to delete bucket: %v", err)
}
f.clearBucketID()
f.clearUploadURL()
return nil
}
// Precision of the remote
func (f *Fs) Precision() time.Duration {
return fs.ModTimeNotSupported
}
// deleteByID deletes a file version given Name and ID
func (f *Fs) deleteByID(ID, Name string) error {
opts := rest.Opts{
Method: "POST",
Path: "/b2_delete_file_version",
}
var request = api.DeleteFileRequest{
ID: ID,
Name: Name,
}
var response api.File
_, err := f.srv.CallJSON(&opts, &request, &response)
if err != nil {
return fmt.Errorf("Failed to delete %q: %v", Name, err)
}
return nil
}
// Purge deletes all the files and directories
//
// Implemented here so we can make sure we delete old versions.
func (f *Fs) Purge() error {
var errReturn error
var checkErrMutex sync.Mutex
var checkErr = func(err error) {
if err == nil {
return
}
checkErrMutex.Lock()
defer checkErrMutex.Unlock()
fs.Stats.Error()
fs.ErrorLog(f, "Purge error: %v", err)
if errReturn == nil {
errReturn = err
}
}
// Delete Config.Transfers in parallel
toBeDeleted := make(chan *api.File, fs.Config.Transfers)
var wg sync.WaitGroup
wg.Add(fs.Config.Transfers)
for i := 0; i < fs.Config.Transfers; i++ {
go func() {
defer wg.Done()
for object := range toBeDeleted {
checkErr(f.deleteByID(object.ID, object.Name))
}
}()
}
checkErr(f.list("", 0, true, func(remote string, object *api.File) error {
fs.Debug(remote, "Deleting (id %q)", object.ID)
toBeDeleted <- object
return nil
}))
close(toBeDeleted)
wg.Wait()
checkErr(f.Rmdir())
return errReturn
}
// ------------------------------------------------------------
// Fs returns the parent Fs
func (o *Object) Fs() fs.Fs {
return o.fs
}
// Return a string version
func (o *Object) String() string {
if o == nil {
return "<nil>"
}
return o.remote
}
// Remote returns the remote path
func (o *Object) Remote() string {
return o.remote
}
// Md5sum returns the Md5sum of an object returning a lowercase hex string
func (o *Object) Md5sum() (string, error) {
return "", nil
}
// Size returns the size of an object in bytes
func (o *Object) Size() int64 {
return o.info.Size
}
// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
func (o *Object) readMetaData() (err error) {
if o.info.ID != "" {
return nil
}
err = o.fs.list(o.remote, 1, false, func(remote string, object *api.File) error {
if remote == "" {
o.info = *object
}
return nil
})
if o.info.ID != "" {
return nil
}
return fmt.Errorf("Object %q not found", o.remote)
}
// timeString returns modTime as the number of milliseconds
// elapsed since January 1, 1970 UTC as a decimal string.
func timeString(modTime time.Time) string {
return strconv.FormatInt(modTime.UnixNano()/1E6, 10)
}
// parseTimeString converts a decimal string number of milliseconds
// elapsed since January 1, 1970 UTC into a time.Time
func parseTimeString(timeString string) (result time.Time, err error) {
if timeString == "" {
return result, fmt.Errorf("%q not found in metadata", timeKey)
}
unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
if err != nil {
return result, err
}
return time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC(), nil
}
// ModTime returns the modification time of the object
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *Object) ModTime() (result time.Time) {
if !o.modTime.IsZero() {
return o.modTime
}
// Return the current time if can't read metadata
result = time.Now()
// Read metadata (need ID)
err := o.readMetaData()
if err != nil {
fs.Debug(o, "Failed to read metadata: %v", err)
return result
}
// Return the UploadTimestamp if can't get file info
result = time.Time(o.info.UploadTimestamp)
// Now read the metadata for the modified time
opts := rest.Opts{
Method: "POST",
Path: "/b2_get_file_info",
}
var request = api.GetFileInfoRequest{
ID: o.info.ID,
}
var response api.FileInfo
_, err = o.fs.srv.CallJSON(&opts, &request, &response)
if err != nil {
fs.Debug(o, "Failed to get file info: %v", err)
return result
}
// Parse the result
timeString := response.Info[timeKey]
parsed, err := parseTimeString(timeString)
if err != nil {
fs.Debug(o, "Failed to parse mod time string %q: %v", timeString, err)
return result
}
return parsed
}
// SetModTime sets the modification time of the local fs object
func (o *Object) SetModTime(modTime time.Time) {
// Not possible with B2
}
// Storable returns if this object is storable
func (o *Object) Storable() bool {
return true
}
// openFile represents an Object open for reading
type openFile struct {
o *Object // Object we are reading for
resp *http.Response // response of the GET
body io.Reader // reading from here
hash hash.Hash // currently accumulating SHA1
bytes int64 // number of bytes read on this connection
eof bool // whether we have read end of file
}
// newOpenFile wraps an io.ReadCloser and checks the sha1sum
func newOpenFile(o *Object, resp *http.Response) *openFile {
file := &openFile{
o: o,
resp: resp,
hash: sha1.New(),
}
file.body = io.TeeReader(resp.Body, file.hash)
return file
}
// Read bytes from the object - see io.Reader
func (file *openFile) Read(p []byte) (n int, err error) {
n, err = file.body.Read(p)
file.bytes += int64(n)
if err == io.EOF {
file.eof = true
}
return
}
// Close the object and checks the length and SHA1 if all the object
// was read
func (file *openFile) Close() (err error) {
// Close the body at the end
defer fs.CheckClose(file.resp.Body, &err)
// If not end of file then can't check SHA1
if !file.eof {
return nil
}
// Check to see we read the correct number of bytes
if file.o.Size() != file.bytes {
return fmt.Errorf("Object corrupted on transfer - length mismatch (want %d got %d)", file.o.Size(), file.bytes)
}
// Check the SHA1
receivedSHA1 := file.resp.Header.Get(sha1Header)
calculatedSHA1 := fmt.Sprintf("%x", file.hash.Sum(nil))
if receivedSHA1 != calculatedSHA1 {
return fmt.Errorf("Object corrupted on transfer - SHA1 mismatch (want %q got %q)", receivedSHA1, calculatedSHA1)
}
return nil
}
// Check it satisfies the interfaces
var _ io.ReadCloser = &openFile{}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
opts := rest.Opts{
Method: "GET",
Absolute: true,
Path: o.fs.info.DownloadURL + "/file/" + urlEncode(o.fs.bucket) + "/" + urlEncode(o.fs.root+o.remote),
}
resp, err := o.fs.srv.Call(&opts)
if err != nil {
return nil, fmt.Errorf("Failed to open for download: %v", err)
}
// Parse the time out of the headers if possible
timeString := resp.Header.Get(timeHeader)
parsed, err := parseTimeString(timeString)
if err != nil {
fs.Debug(o, "Failed to parse mod time string %q: %v", timeString, err)
} else {
o.modTime = parsed
}
return newOpenFile(o, resp), nil
}
// dontEncode is the characters that do not need percent-encoding
//
// The characters that do not need percent-encoding are a subset of
// the printable ASCII characters: upper-case letters, lower-case
// letters, digits, ".", "_", "-", "/", "~", "!", "$", "'", "(", ")",
// "*", ";", "=", ":", and "@". All other byte values in a UTF-8 must
// be replaced with "%" and the two-digit hex value of the byte.
const dontEncode = (`abcdefghijklmnopqrstuvwxyz` +
`ABCDEFGHIJKLMNOPQRSTUVWXYZ` +
`0123456789` +
`._-/~!$'()*;=:@`)
// noNeedToEncode is a bitmap of characters which don't need % encoding
var noNeedToEncode [256]bool
func init() {
for _, c := range dontEncode {
noNeedToEncode[c] = true
}
}
// urlEncode encodes in with % encoding
func urlEncode(in string) string {
var out bytes.Buffer
for i := 0; i < len(in); i++ {
c := in[i]
if noNeedToEncode[c] {
_ = out.WriteByte(c)
} else {
_, _ = out.WriteString(fmt.Sprintf("%%%2X", c))
}
}
return out.String()
}
// Update the object with the contents of the io.Reader, modTime and size
//
// The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) (err error) {
// Open a temp file to copy the input
fd, err := ioutil.TempFile("", "rclone-b2-")
if err != nil {
return err
}
_ = os.Remove(fd.Name()) // Delete the file - may not work on Windows
defer func() {
_ = fd.Close() // Ignore error may have been closed already
_ = os.Remove(fd.Name()) // Delete the file - may have been deleted already
}()
// Copy the input while calculating the sha1
hash := sha1.New()
teed := io.TeeReader(in, hash)
n, err := io.Copy(fd, teed)
if err != nil {
return err
}
if n != size {
return fmt.Errorf("Read %d bytes expecting %d", n, size)
}
calculatedSha1 := fmt.Sprintf("%x", hash.Sum(nil))
// Rewind the temporary file
_, err = fd.Seek(0, 0)
if err != nil {
return err
}
// Get upload URL
UploadURL, AuthorizationToken, err := o.fs.getUploadURL()
if err != nil {
return err
}
// Headers for upload file
//
// Authorization
// required
// An upload authorization token, from b2_get_upload_url.
//
// X-Bz-File-Name
// required
//
// The name of the file, in percent-encoded UTF-8. See Files for requirements on file names. See String Encoding.
//
// Content-Type
// required
//
// The MIME type of the content of the file, which will be returned in
// the Content-Type header when downloading the file. Use the
// Content-Type b2/x-auto to automatically set the stored Content-Type
// post upload. In the case where a file extension is absent or the
// lookup fails, the Content-Type is set to application/octet-stream. The
// Content-Type mappings can be purused here.
//
// X-Bz-Content-Sha1
// required
//
// The SHA1 checksum of the content of the file. B2 will check this when
// the file is uploaded, to make sure that the file arrived correctly. It
// will be returned in the X-Bz-Content-Sha1 header when the file is
// downloaded.
//
// X-Bz-Info-src_last_modified_millis
// optional
//
// If the original source of the file being uploaded has a last modified
// time concept, Backblaze recommends using this spelling of one of your
// ten X-Bz-Info-* headers (see below). Using a standard spelling allows
// different B2 clients and the B2 web user interface to interoperate
// correctly. The value should be a base 10 number which represents a UTC
// time when the original source file was last modified. It is a base 10
// number of milliseconds since midnight, January 1, 1970 UTC. This fits
// in a 64 bit integer such as the type "long" in the programming
// language Java. It is intended to be compatible with Java's time
// long. For example, it can be passed directly into the Java call
// Date.setTime(long time).
//
// X-Bz-Info-*
// optional
//
// Up to 10 of these headers may be present. The * part of the header
// name is replace with the name of a custom field in the file
// information stored with the file, and the value is an arbitrary UTF-8
// string, percent-encoded. The same info headers sent with the upload
// will be returned with the download.
opts := rest.Opts{
Method: "POST",
Absolute: true,
Path: UploadURL,
Body: fd,
ExtraHeaders: map[string]string{
"Authorization": AuthorizationToken,
"X-Bz-File-Name": urlEncode(o.fs.root + o.remote),
"Content-Type": fs.MimeType(o),
sha1Header: calculatedSha1,
timeHeader: timeString(modTime),
},
ContentLength: &size,
}
var response api.FileInfo
_, err = o.fs.srv.CallJSON(&opts, nil, &response)
if err != nil {
return fmt.Errorf("Failed to upload: %v", err)
}
o.info.ID = response.ID
o.info.Name = response.Name
o.info.Action = "upload"
o.info.Size = response.Size
o.info.UploadTimestamp = api.Timestamp(time.Now()) // FIXME not quite right
return nil
}
// Remove an object
func (o *Object) Remove() error {
bucketID, err := o.fs.getBucketID()
if err != nil {
return err
}
opts := rest.Opts{
Method: "POST",
Path: "/b2_hide_file",
}
var request = api.HideFileRequest{
BucketID: bucketID,
Name: o.info.Name,
}
var response api.File
_, err = o.fs.srv.CallJSON(&opts, &request, &response)
if err != nil {
return fmt.Errorf("Failed to delete file: %v", err)
}
return nil
}
// Check the interfaces are satisfied
var (
_ fs.Fs = &Fs{}
_ fs.Purger = &Fs{}
_ fs.Object = &Object{}
)

168
b2/b2_internal_test.go Normal file
View file

@ -0,0 +1,168 @@
package b2
import (
"testing"
"time"
"github.com/ncw/rclone/fstest"
)
// Test b2 string encoding
// https://www.backblaze.com/b2/docs/string_encoding.html
var encodeTest = []struct {
fullyEncoded string
minimallyEncoded string
plainText string
}{
{fullyEncoded: "%20", minimallyEncoded: "+", plainText: " "},
{fullyEncoded: "%21", minimallyEncoded: "!", plainText: "!"},
{fullyEncoded: "%22", minimallyEncoded: "%22", plainText: "\""},
{fullyEncoded: "%23", minimallyEncoded: "%23", plainText: "#"},
{fullyEncoded: "%24", minimallyEncoded: "$", plainText: "$"},
{fullyEncoded: "%25", minimallyEncoded: "%25", plainText: "%"},
{fullyEncoded: "%26", minimallyEncoded: "%26", plainText: "&"},
{fullyEncoded: "%27", minimallyEncoded: "'", plainText: "'"},
{fullyEncoded: "%28", minimallyEncoded: "(", plainText: "("},
{fullyEncoded: "%29", minimallyEncoded: ")", plainText: ")"},
{fullyEncoded: "%2A", minimallyEncoded: "*", plainText: "*"},
{fullyEncoded: "%2B", minimallyEncoded: "%2B", plainText: "+"},
{fullyEncoded: "%2C", minimallyEncoded: "%2C", plainText: ","},
{fullyEncoded: "%2D", minimallyEncoded: "-", plainText: "-"},
{fullyEncoded: "%2E", minimallyEncoded: ".", plainText: "."},
{fullyEncoded: "%2F", minimallyEncoded: "/", plainText: "/"},
{fullyEncoded: "%30", minimallyEncoded: "0", plainText: "0"},
{fullyEncoded: "%31", minimallyEncoded: "1", plainText: "1"},
{fullyEncoded: "%32", minimallyEncoded: "2", plainText: "2"},
{fullyEncoded: "%33", minimallyEncoded: "3", plainText: "3"},
{fullyEncoded: "%34", minimallyEncoded: "4", plainText: "4"},
{fullyEncoded: "%35", minimallyEncoded: "5", plainText: "5"},
{fullyEncoded: "%36", minimallyEncoded: "6", plainText: "6"},
{fullyEncoded: "%37", minimallyEncoded: "7", plainText: "7"},
{fullyEncoded: "%38", minimallyEncoded: "8", plainText: "8"},
{fullyEncoded: "%39", minimallyEncoded: "9", plainText: "9"},
{fullyEncoded: "%3A", minimallyEncoded: ":", plainText: ":"},
{fullyEncoded: "%3B", minimallyEncoded: ";", plainText: ";"},
{fullyEncoded: "%3C", minimallyEncoded: "%3C", plainText: "<"},
{fullyEncoded: "%3D", minimallyEncoded: "=", plainText: "="},
{fullyEncoded: "%3E", minimallyEncoded: "%3E", plainText: ">"},
{fullyEncoded: "%3F", minimallyEncoded: "%3F", plainText: "?"},
{fullyEncoded: "%40", minimallyEncoded: "@", plainText: "@"},
{fullyEncoded: "%41", minimallyEncoded: "A", plainText: "A"},
{fullyEncoded: "%42", minimallyEncoded: "B", plainText: "B"},
{fullyEncoded: "%43", minimallyEncoded: "C", plainText: "C"},
{fullyEncoded: "%44", minimallyEncoded: "D", plainText: "D"},
{fullyEncoded: "%45", minimallyEncoded: "E", plainText: "E"},
{fullyEncoded: "%46", minimallyEncoded: "F", plainText: "F"},
{fullyEncoded: "%47", minimallyEncoded: "G", plainText: "G"},
{fullyEncoded: "%48", minimallyEncoded: "H", plainText: "H"},
{fullyEncoded: "%49", minimallyEncoded: "I", plainText: "I"},
{fullyEncoded: "%4A", minimallyEncoded: "J", plainText: "J"},
{fullyEncoded: "%4B", minimallyEncoded: "K", plainText: "K"},
{fullyEncoded: "%4C", minimallyEncoded: "L", plainText: "L"},
{fullyEncoded: "%4D", minimallyEncoded: "M", plainText: "M"},
{fullyEncoded: "%4E", minimallyEncoded: "N", plainText: "N"},
{fullyEncoded: "%4F", minimallyEncoded: "O", plainText: "O"},
{fullyEncoded: "%50", minimallyEncoded: "P", plainText: "P"},
{fullyEncoded: "%51", minimallyEncoded: "Q", plainText: "Q"},
{fullyEncoded: "%52", minimallyEncoded: "R", plainText: "R"},
{fullyEncoded: "%53", minimallyEncoded: "S", plainText: "S"},
{fullyEncoded: "%54", minimallyEncoded: "T", plainText: "T"},
{fullyEncoded: "%55", minimallyEncoded: "U", plainText: "U"},
{fullyEncoded: "%56", minimallyEncoded: "V", plainText: "V"},
{fullyEncoded: "%57", minimallyEncoded: "W", plainText: "W"},
{fullyEncoded: "%58", minimallyEncoded: "X", plainText: "X"},
{fullyEncoded: "%59", minimallyEncoded: "Y", plainText: "Y"},
{fullyEncoded: "%5A", minimallyEncoded: "Z", plainText: "Z"},
{fullyEncoded: "%5B", minimallyEncoded: "%5B", plainText: "["},
{fullyEncoded: "%5C", minimallyEncoded: "%5C", plainText: "\\"},
{fullyEncoded: "%5D", minimallyEncoded: "%5D", plainText: "]"},
{fullyEncoded: "%5E", minimallyEncoded: "%5E", plainText: "^"},
{fullyEncoded: "%5F", minimallyEncoded: "_", plainText: "_"},
{fullyEncoded: "%60", minimallyEncoded: "%60", plainText: "`"},
{fullyEncoded: "%61", minimallyEncoded: "a", plainText: "a"},
{fullyEncoded: "%62", minimallyEncoded: "b", plainText: "b"},
{fullyEncoded: "%63", minimallyEncoded: "c", plainText: "c"},
{fullyEncoded: "%64", minimallyEncoded: "d", plainText: "d"},
{fullyEncoded: "%65", minimallyEncoded: "e", plainText: "e"},
{fullyEncoded: "%66", minimallyEncoded: "f", plainText: "f"},
{fullyEncoded: "%67", minimallyEncoded: "g", plainText: "g"},
{fullyEncoded: "%68", minimallyEncoded: "h", plainText: "h"},
{fullyEncoded: "%69", minimallyEncoded: "i", plainText: "i"},
{fullyEncoded: "%6A", minimallyEncoded: "j", plainText: "j"},
{fullyEncoded: "%6B", minimallyEncoded: "k", plainText: "k"},
{fullyEncoded: "%6C", minimallyEncoded: "l", plainText: "l"},
{fullyEncoded: "%6D", minimallyEncoded: "m", plainText: "m"},
{fullyEncoded: "%6E", minimallyEncoded: "n", plainText: "n"},
{fullyEncoded: "%6F", minimallyEncoded: "o", plainText: "o"},
{fullyEncoded: "%70", minimallyEncoded: "p", plainText: "p"},
{fullyEncoded: "%71", minimallyEncoded: "q", plainText: "q"},
{fullyEncoded: "%72", minimallyEncoded: "r", plainText: "r"},
{fullyEncoded: "%73", minimallyEncoded: "s", plainText: "s"},
{fullyEncoded: "%74", minimallyEncoded: "t", plainText: "t"},
{fullyEncoded: "%75", minimallyEncoded: "u", plainText: "u"},
{fullyEncoded: "%76", minimallyEncoded: "v", plainText: "v"},
{fullyEncoded: "%77", minimallyEncoded: "w", plainText: "w"},
{fullyEncoded: "%78", minimallyEncoded: "x", plainText: "x"},
{fullyEncoded: "%79", minimallyEncoded: "y", plainText: "y"},
{fullyEncoded: "%7A", minimallyEncoded: "z", plainText: "z"},
{fullyEncoded: "%7B", minimallyEncoded: "%7B", plainText: "{"},
{fullyEncoded: "%7C", minimallyEncoded: "%7C", plainText: "|"},
{fullyEncoded: "%7D", minimallyEncoded: "%7D", plainText: "}"},
{fullyEncoded: "%7E", minimallyEncoded: "~", plainText: "~"},
{fullyEncoded: "%7F", minimallyEncoded: "%7F", plainText: "\u007f"},
{fullyEncoded: "%E8%87%AA%E7%94%B1", minimallyEncoded: "%E8%87%AA%E7%94%B1", plainText: "自由"},
{fullyEncoded: "%F0%90%90%80", minimallyEncoded: "%F0%90%90%80", plainText: "𐐀"},
}
func TestUrlEncode(t *testing.T) {
for _, test := range encodeTest {
got := urlEncode(test.plainText)
if got != test.minimallyEncoded && got != test.fullyEncoded {
t.Errorf("urlEncode(%q) got %q wanted %q or %q", test.plainText, got, test.minimallyEncoded, test.fullyEncoded)
}
}
}
func TestTimeString(t *testing.T) {
for _, test := range []struct {
in time.Time
want string
}{
{fstest.Time("1970-01-01T00:00:00.000000000Z"), "0"},
{fstest.Time("2001-02-03T04:05:10.123123123Z"), "981173110123"},
{fstest.Time("2001-02-03T05:05:10.123123123+01:00"), "981173110123"},
} {
got := timeString(test.in)
if test.want != got {
t.Logf("%v: want %v got %v", test.in, test.want, got)
}
}
}
func TestParseTimeString(t *testing.T) {
for _, test := range []struct {
in string
want time.Time
wantError string
}{
{"0", fstest.Time("1970-01-01T00:00:00.000000000Z"), ""},
{"981173110123", fstest.Time("2001-02-03T04:05:10.123000000Z"), ""},
{"", time.Time{}, `"src_last_modified_millis" not found in metadata`},
{"potato", time.Time{}, `strconv.ParseInt: parsing "potato": invalid syntax`},
} {
got, err := parseTimeString(test.in)
var gotError string
if err != nil {
gotError = err.Error()
}
if test.want != got {
t.Logf("%v: want %v got %v", test.in, test.want, got)
}
if test.wantError != gotError {
t.Logf("%v: want error %v got error %v", test.in, test.wantError, gotError)
}
}
}

56
b2/b2_test.go Normal file
View file

@ -0,0 +1,56 @@
// Test B2 filesystem interface
//
// Automatically generated - DO NOT EDIT
// Regenerate with: make gen_tests
package b2_test
import (
"testing"
"github.com/ncw/rclone/b2"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
)
func init() {
fstests.NilObject = fs.Object((*b2.Object)(nil))
fstests.RemoteName = "TestB2:"
}
// Generic tests for the Fs
func TestInit(t *testing.T) { fstests.TestInit(t) }
func TestFsString(t *testing.T) { fstests.TestFsString(t) }
func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) }
func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) }
func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsMove(t *testing.T) { fstests.TestFsMove(t) }
func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }

1
docs/content/about.md
View file

@ -21,6 +21,7 @@ Rclone is a command line program to sync files and directories to and from
* Amazon Cloud Drive * Amazon Cloud Drive
* Microsoft One Drive * Microsoft One Drive
* Hubic * Hubic
* Backblaze B2
* The local filesystem * The local filesystem
Features Features

124
docs/content/b2.md Normal file
View file

@ -0,0 +1,124 @@
---
title: "B2"
description: "Backblaze B2"
date: "2015-12-29"
---
<i class="fa fa-fire"></i>Backblaze B2
----------------------------------------
B2 is [Backblaze's cloud storage system](https://www.backblaze.com/b2/).
Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
command.) You may put subdirectories in too, eg `remote:bucket/path/to/dir`.
Here is an example of making a b2 configuration. First run
rclone config
This will guide you through an interactive setup process. You will
need your account number (a short hex number) and key (a long hex
number) which you can get from the b2 control panel.
```
No remotes found - make a new one
n) New remote
q) Quit config
n/q> n
name> remote
What type of source is it?
Choose a number from below
1) amazon cloud drive
2) b2
3) drive
4) dropbox
5) google cloud storage
6) swift
7) hubic
8) local
9) onedrive
10) s3
type> 2
Account ID
account> 123456789abc
Application Key
key> 0123456789abcdef0123456789abcdef0123456789
Endpoint for the service - leave blank normally.
endpoint>
Remote config
--------------------
[remote]
account = 123456789abc
key = 0123456789abcdef0123456789abcdef0123456789
endpoint =
--------------------
y) Yes this is OK
e) Edit this remote
d) Delete this remote
y/e/d> y
```
This remote is called `remote` and can now be used like this
See all buckets
rclone lsd remote:
Make a new bucket
rclone mkdir remote:bucket
List the contents of a bucket
rclone ls remote:bucket
Sync `/home/local/directory` to the remote bucket, deleting any
excess files in the bucket.
rclone sync /home/local/directory remote:bucket
### Modified time ###
The modified time is stored as metadata on the object as
`X-Bz-Info-src_last_modified_millis` as milliseconds since 1970-01-01
in the Backblaze standard. Other tools should be able to use this as
a modified time.
Modified times are set on upload, read on download and shown in
listings. They are not used in syncing as unfortunately B2 doesn't
have an API method to set them independently of doing an upload.
### SHA1 checksums ###
The SHA1 checksums of the files are checked on upload and download,
but they aren't used for sync as they aren't compatible with the MD5
checksums rclone normally uses.
### Versions ###
When rclone uploads a new version of a file it creates a [new version
of it](https://www.backblaze.com/b2/docs/file_versions.html).
Likewise when you delete a file, the old version will still be
available.
The old versions of files are visible in the B2 web interface, but not
via rclone yet.
Rclone doesn't provide any way of managing old versions (downloading
them or deleting them) at the moment. When you `purge` a bucket, all
the old versions will be deleted.
### Bugs ###
Note that when uploading a file, rclone has to make a temporary copy
of it in your temp filing system. This is due to a weakness in the B2
API which I'm hoping will be addressed soon.
### API ###
Here are [some notes I made on the backblaze
API](https://gist.github.com/ncw/166dabf352b399f1cc1c) while
integrating it with rclone which detail the changes I'd like to see.
With a couple of small tweaks Backblaze could enable rclone to not
make a temporary copy of all files and fully support modification
times.

4
docs/content/overview.md
View file

@ -25,6 +25,7 @@ Here is an overview of the major features of each cloud storage system.
| Amazon Cloud Drive | Yes | No | Yes | No | | Amazon Cloud Drive | Yes | No | Yes | No |
| Microsoft One Drive | No | Yes | Yes | No | | Microsoft One Drive | No | Yes | Yes | No |
| Hubic | Yes | Yes | No | No | | Hubic | Yes | Yes | No | No |
| Backblaze B2 | No | Partial | No | No |
| The local filesystem | Yes | Yes | Depends | No | | The local filesystem | Yes | Yes | Depends | No |
### MD5SUM ### ### MD5SUM ###
@ -44,6 +45,9 @@ default, though the MD5SUM can be checked with the `--checksum` flag.
All cloud storage systems support some kind of date on the object and All cloud storage systems support some kind of date on the object and
these will be set when transferring from the cloud storage system. these will be set when transferring from the cloud storage system.
Backblaze B2 preserves file modification times on files uploaded and
downloaded, but doesn't use them to decide which objects to sync.
### Case Insensitive ### ### Case Insensitive ###
If a cloud storage systems is case sensitive then it is possible to If a cloud storage systems is case sensitive then it is possible to

1
docs/layouts/chrome/navbar.html
View file

@ -39,6 +39,7 @@
<li><a href="/amazonclouddrive/"><i class="fa fa-amazon"></i> Amazon Cloud Drive</a></li> <li><a href="/amazonclouddrive/"><i class="fa fa-amazon"></i> Amazon Cloud Drive</a></li>
<li><a href="/onedrive/"><i class="fa fa-windows"></i> Microsoft One Drive</a></li> <li><a href="/onedrive/"><i class="fa fa-windows"></i> Microsoft One Drive</a></li>
<li><a href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a></li> <li><a href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a></li>
<li><a href="/b2/"><i class="fa fa-fire"></i> Backblaze B2</a></li>
<li><a href="/local/"><i class="fa fa-file"></i> Local</a></li> <li><a href="/local/"><i class="fa fa-file"></i> Local</a></li>
</ul> </ul>
</li> </li>

1
fs/operations_test.go
View file

@ -21,6 +21,7 @@ import (
// Active file systems // Active file systems
_ "github.com/ncw/rclone/amazonclouddrive" _ "github.com/ncw/rclone/amazonclouddrive"
_ "github.com/ncw/rclone/b2"
_ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/googlecloudstorage" _ "github.com/ncw/rclone/googlecloudstorage"

1
fs/test_all.go
View file

@ -25,6 +25,7 @@ var (
"TestAmazonCloudDrive:", "TestAmazonCloudDrive:",
"TestOneDrive:", "TestOneDrive:",
"TestHubic:", "TestHubic:",
"TestB2:",
} }
binary = "fs.test" binary = "fs.test"
// Flags // Flags

1
fstest/fstests/gen_tests.go
View file

@ -133,5 +133,6 @@ func main() {
generateTestProgram(t, fns, "AmazonCloudDrive") generateTestProgram(t, fns, "AmazonCloudDrive")
generateTestProgram(t, fns, "OneDrive") generateTestProgram(t, fns, "OneDrive")
generateTestProgram(t, fns, "Hubic") generateTestProgram(t, fns, "Hubic")
generateTestProgram(t, fns, "B2")
log.Printf("Done") log.Printf("Done")
} }

1
make_manual.py
View file

@ -26,6 +26,7 @@ docs = [
"amazonclouddrive.md", "amazonclouddrive.md",
"onedrive.md", "onedrive.md",
"hubic.md", "hubic.md",
"b2.md",
"local.md", "local.md",
"changelog.md", "changelog.md",
"bugs.md", "bugs.md",

2
onedrive/onedrive.go
View file

@ -177,7 +177,7 @@ func NewFs(name, root string) (fs.Fs, error) {
f := &Fs{ f := &Fs{
name: name, name: name,
root: root, root: root,
srv: rest.NewClient(oAuthClient, rootURL), srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
} }
f.srv.SetErrorHandler(errorHandler) f.srv.SetErrorHandler(errorHandler)

1
rclone.go
View file

@ -17,6 +17,7 @@ import (
"github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs"
// Active file systems // Active file systems
_ "github.com/ncw/rclone/amazonclouddrive" _ "github.com/ncw/rclone/amazonclouddrive"
_ "github.com/ncw/rclone/b2"
_ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/googlecloudstorage" _ "github.com/ncw/rclone/googlecloudstorage"

61
rest/rest.go
View file

@ -6,6 +6,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs"
@ -16,27 +17,48 @@ type Client struct {
c *http.Client c *http.Client
rootURL string rootURL string
errorHandler func(resp *http.Response) error errorHandler func(resp *http.Response) error
headers map[string]string
} }
// NewClient takes an oauth http.Client and makes a new api instance // NewClient takes an oauth http.Client and makes a new api instance
func NewClient(c *http.Client, rootURL string) *Client { func NewClient(c *http.Client) *Client {
return &Client{ api := &Client{
c: c, c: c,
rootURL: rootURL,
errorHandler: defaultErrorHandler, errorHandler: defaultErrorHandler,
headers: make(map[string]string),
} }
api.SetHeader("User-Agent", fs.UserAgent)
return api
} }
// defaultErrorHandler doesn't attempt to parse the http body // defaultErrorHandler doesn't attempt to parse the http body, just
// returns it in the error message
func defaultErrorHandler(resp *http.Response) (err error) { func defaultErrorHandler(resp *http.Response) (err error) {
defer fs.CheckClose(resp.Body, &err) defer fs.CheckClose(resp.Body, &err)
return fmt.Errorf("HTTP error %v (%v) returned", resp.StatusCode, resp.Status) body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return fmt.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body)
} }
// SetErrorHandler sets the handler to decode an error response when // SetErrorHandler sets the handler to decode an error response when
// the HTTP status code is not 2xx. The handler should close resp.Body. // the HTTP status code is not 2xx. The handler should close resp.Body.
func (api *Client) SetErrorHandler(fn func(resp *http.Response) error) { func (api *Client) SetErrorHandler(fn func(resp *http.Response) error) *Client {
api.errorHandler = fn api.errorHandler = fn
return api
}
// SetRoot sets the default root URL
func (api *Client) SetRoot(RootURL string) *Client {
api.rootURL = RootURL
return api
}
// SetHeader sets a header for all requests
func (api *Client) SetHeader(key, value string) *Client {
api.headers[key] = value
return api
} }
// Opts contains parameters for Call, CallJSON etc // Opts contains parameters for Call, CallJSON etc
@ -50,6 +72,8 @@ type Opts struct {
ContentLength *int64 ContentLength *int64
ContentRange string ContentRange string
ExtraHeaders map[string]string ExtraHeaders map[string]string
UserName string // username for Basic Auth
Password string // password for Basic Auth
} }
// DecodeJSON decodes resp.Body into result // DecodeJSON decodes resp.Body into result
@ -72,27 +96,42 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
if opts.Absolute { if opts.Absolute {
url = opts.Path url = opts.Path
} else { } else {
if api.rootURL == "" {
return nil, fmt.Errorf("RootURL not set")
}
url = api.rootURL + opts.Path url = api.rootURL + opts.Path
} }
req, err := http.NewRequest(opts.Method, url, opts.Body) req, err := http.NewRequest(opts.Method, url, opts.Body)
if err != nil { if err != nil {
return return
} }
headers := make(map[string]string)
// Set default headers
for k, v := range api.headers {
headers[k] = v
}
if opts.ContentType != "" { if opts.ContentType != "" {
req.Header.Add("Content-Type", opts.ContentType) headers["Content-Type"] = opts.ContentType
} }
if opts.ContentLength != nil { if opts.ContentLength != nil {
req.ContentLength = *opts.ContentLength req.ContentLength = *opts.ContentLength
} }
if opts.ContentRange != "" { if opts.ContentRange != "" {
req.Header.Add("Content-Range", opts.ContentRange) headers["Content-Range"] = opts.ContentRange
} }
// Set any extra headers
if opts.ExtraHeaders != nil { if opts.ExtraHeaders != nil {
for k, v := range opts.ExtraHeaders { for k, v := range opts.ExtraHeaders {
req.Header.Add(k, v) headers[k] = v
} }
} }
req.Header.Add("User-Agent", fs.UserAgent) // Now set the headers
for k, v := range headers {
req.Header.Add(k, v)
}
if opts.UserName != "" || opts.Password != "" {
req.SetBasicAuth(opts.UserName, opts.Password)
}
resp, err = api.c.Do(req) resp, err = api.c.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
@ -127,7 +166,7 @@ func (api *Client) CallJSON(opts *Opts, request interface{}, response interface{
if err != nil { if err != nil {
return resp, err return resp, err
} }
if opts.NoResponse { if response == nil || opts.NoResponse {
return resp, nil return resp, nil
} }
err = DecodeJSON(resp, response) err = DecodeJSON(resp, response)