From 20f4b2c91da73f31882debcff3c5473eed27961b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 22 Mar 2016 14:39:56 +0000 Subject: [PATCH] b2: update API to new version - fixes #393 * Make reading mod time and SHA1 much more efficient * removes an HTTP transaction to increase speed * Reduce memory usage of the objects --- b2/api/types.go | 14 ++++-- b2/b2.go | 128 ++++++++++++++++++------------------------------ 2 files changed, 58 insertions(+), 84 deletions(-) diff --git a/b2/api/types.go b/b2/api/types.go index 9221c2533..a9734d7a1 100644 --- a/b2/api/types.go +++ b/b2/api/types.go @@ -57,11 +57,14 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error { // File is info about a file type File struct { - ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version. - Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name. - Action string `json:"action"` // Either "upload" or "hide". "upload" means a file that was uploaded to B2 Cloud Storage. "hide" means a file version marking the file as hidden, so that it will not show up in b2_list_file_names. The result of b2_list_file_names will contain only "upload". The result of b2_list_file_versions may have both. - Size int64 `json:"size"` // The number of bytes in the file. - UploadTimestamp Timestamp `json:"uploadTimestamp"` // This is a UTC time when this file was uploaded. + ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version. + Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name. + Action string `json:"action"` // Either "upload" or "hide". "upload" means a file that was uploaded to B2 Cloud Storage. "hide" means a file version marking the file as hidden, so that it will not show up in b2_list_file_names. The result of b2_list_file_names will contain only "upload". The result of b2_list_file_versions may have both. + Size int64 `json:"size"` // The number of bytes in the file. + UploadTimestamp Timestamp `json:"uploadTimestamp"` // This is a UTC time when this file was uploaded. + SHA1 string `json:"contentSha1"` // The SHA1 of the bytes stored in the file. + ContentType string `json:"contentType"` // The MIME type of the file. + Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file. } // AuthorizeAccountResponse is as returned from the b2_authorize_account call @@ -108,6 +111,7 @@ type GetUploadURLResponse struct { type FileInfo struct { ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version. Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name. + Action string `json:"action"` // Either "upload" or "hide". "upload" means a file that was uploaded to B2 Cloud Storage. "hide" means a file version marking the file as hidden, so that it will not show up in b2_list_file_names. The result of b2_list_file_names will contain only "upload". The result of b2_list_file_versions may have both. AccountID string `json:"accountId"` // Your account ID. BucketID string `json:"bucketId"` // The bucket that the file is in. Size int64 `json:"contentLength"` // The number of bytes stored in the file. diff --git a/b2/b2.go b/b2/b2.go index 8f8bffbaa..395ce0953 100644 --- a/b2/b2.go +++ b/b2/b2.go @@ -80,14 +80,13 @@ type Fs struct { } // Object describes a b2 object -// -// Will definitely have info type Object struct { fs *Fs // what this object is part of remote string // The remote path - info api.File // Info from the b2 object if known + id string // b2 id of the file modTime time.Time // The modified time of the object if known sha1 string // SHA-1 hash if known + size int64 // Size of the object } // ------------------------------------------------------------ @@ -308,8 +307,11 @@ func (f *Fs) newFsObjectWithInfo(remote string, info *api.File) fs.Object { remote: remote, } if info != nil { - // Set info but not headers - o.info = *info + err := o.decodeMetaData(info) + if err != nil { + fs.Debug(o, "Failed to decode metadata: %s", err) + return nil + } } else { err := o.readMetaData() // reads info and headers, returning an error if err != nil { @@ -375,9 +377,6 @@ func (f *Fs) list(prefix string, limit int, hidden bool, fn listFn) error { return f.shouldRetry(resp, err) }) if err != nil { - if err == errEndList { - return nil - } return err } for i := range response.Files { @@ -388,6 +387,9 @@ func (f *Fs) list(prefix string, limit int, hidden bool, fn listFn) error { } err = fn(file.Name[len(f.root):], file) if err != nil { + if err == errEndList { + return nil + } return err } } @@ -711,8 +713,8 @@ func (o *Object) Hash(t fs.HashType) (string, error) { return "", fs.ErrHashUnsupported } if o.sha1 == "" { - // Error is logged in readFileMetadata - err := o.readFileMetadata() + // Error is logged in readMetaData + err := o.readMetaData() if err != nil { return "", err } @@ -722,26 +724,50 @@ func (o *Object) Hash(t fs.HashType) (string, error) { // Size returns the size of an object in bytes func (o *Object) Size() int64 { - return o.info.Size + return o.size +} + +// decodeMetaData sets the metadata in the object from info +// +// Sets +// o.id +// o.modTime +// o.size +// o.sha1 +func (o *Object) decodeMetaData(info *api.File) (err error) { + o.id = info.ID + o.sha1 = info.SHA1 + o.size = info.Size + // Use the UploadTimestamp if can't get file info + o.modTime = time.Time(info.UploadTimestamp) + return o.parseTimeString(info.Info[timeKey]) } // readMetaData gets the metadata if it hasn't already been fetched // -// it also sets the info +// Sets +// o.id +// o.modTime +// o.size +// o.sha1 func (o *Object) readMetaData() (err error) { - if o.info.ID != "" { + if o.id != "" { return nil } + var info *api.File err = o.fs.list(o.remote, 1, false, func(remote string, object *api.File) error { if remote == o.remote { - o.info = *object + info = object } return errEndList // read only 1 item }) - if o.info.ID != "" { - return nil + if err != nil { + return err } - return fmt.Errorf("Object %q not found", o.remote) + if info == nil { + return fmt.Errorf("Object %q not found", o.remote) + } + return o.decodeMetaData(info) } // timeString returns modTime as the number of milliseconds @@ -766,61 +792,6 @@ func (o *Object) parseTimeString(timeString string) (err error) { return nil } -// readFileMetadata attempts to read the modified time and -// SHA-1 hash of the remote object. -// -// If the objects mtime and if that isn't present the -// LastModified returned in the http headers. -// -// It is safe to call this function multiple times, and the -// result is cached between calls. -func (o *Object) readFileMetadata() error { - // Return if already know it - if !o.modTime.IsZero() && o.sha1 != "" { - return nil - } - - // Set modtime to now, as default value. - o.modTime = time.Now() - - // Read metadata (we need the ID) - err := o.readMetaData() - if err != nil { - fs.Debug(o, "Failed to get file metadata: %v", err) - return err - } - - // Use the UploadTimestamp if can't get file info - o.modTime = time.Time(o.info.UploadTimestamp) - - // Now read the metadata for the modified time - opts := rest.Opts{ - Method: "POST", - Path: "/b2_get_file_info", - } - var request = api.GetFileInfoRequest{ - ID: o.info.ID, - } - var response api.FileInfo - err = o.fs.pacer.Call(func() (bool, error) { - resp, err := o.fs.srv.CallJSON(&opts, &request, &response) - return o.fs.shouldRetry(resp, err) - }) - if err != nil { - fs.Debug(o, "Failed to get file info: %v", err) - return err - } - o.sha1 = response.SHA1 - - // Parse the result - err = o.parseTimeString(response.Info[timeKey]) - if err != nil { - return err - } - - return nil -} - // ModTime returns the modification time of the object // // It attempts to read the objects mtime and if that isn't present the @@ -828,8 +799,8 @@ func (o *Object) readFileMetadata() error { // // SHA-1 will also be updated once the request has completed. func (o *Object) ModTime() (result time.Time) { - // The error is logged in readFileMetadata - _ = o.readFileMetadata() + // The error is logged in readMetaData + _ = o.readMetaData() return o.modTime } @@ -1093,12 +1064,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) { if err != nil { return err } - o.info.ID = response.ID - o.info.Name = response.Name - o.info.Action = "upload" - o.info.Size = response.Size - o.info.UploadTimestamp = api.Timestamp(time.Now()) // FIXME not quite right + o.id = response.ID o.sha1 = response.SHA1 + o.size = response.Size + o.modTime = modTime + _ = o.parseTimeString(response.Info[timeKey]) return nil }