From 02afcb00e93e40b48bb55b82b6fc792b3924634d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 18 Apr 2014 17:04:21 +0100 Subject: [PATCH] Factor Fs.Put into Object.Update and call Update rather than Put if possible --- drive/drive.go | 111 +++++++++++++++++++++++++---------------------- fs/fs.go | 3 ++ fs/operations.go | 16 +++++-- local/local.go | 52 ++++++++++++---------- s3/s3.go | 32 ++++++++------ swift/swift.go | 22 ++++++---- 6 files changed, 135 insertions(+), 101 deletions(-) diff --git a/drive/drive.go b/drive/drive.go index 08bd0a758..90dea4348 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -648,63 +648,15 @@ func (f *FsDrive) ListDir() fs.DirChan { return out } -// Put the FsObject into the container +// Put the object // // Copy the reader in to the new object which is returned // -// The new object may have been created +// The new object may have been created if an error is returned func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { // Temporary FsObject under construction fs := &FsObjectDrive{drive: f, remote: remote} - - directory, leaf := splitPath(remote) - directoryId, err := f.findDir(directory, true) - if err != nil { - return nil, fmt.Errorf("Couldn't find or make directory: %s", err) - } - - // See if the file already exists - var info *drive.File - found, err := f.listAll(directoryId, leaf, false, true, func(item *drive.File) bool { - info = item - return true - }) - if err != nil { - return nil, fmt.Errorf("Error finding file: %s", leaf, err) - } - - // Guess the mime type - mimeType := mime.TypeByExtension(path.Ext(remote)) - if mimeType == "" { - mimeType = "application/octet-stream" - } - modifiedDate := modTime.Format(time.RFC3339Nano) - - if found { - // Modify metadata - info.ModifiedDate = modifiedDate - info.MimeType = mimeType - - // Make the API request to upload metadata and file data. - info, err = f.svc.Files.Update(info.Id, info).SetModifiedDate(true).Media(in).Do() - } else { - // Define the metadata for the file we are going to create. - info = &drive.File{ - Title: leaf, - Description: leaf, - Parents: []*drive.ParentReference{{Id: directoryId}}, - MimeType: mimeType, - ModifiedDate: modifiedDate, - } - - // Make the API request to upload metadata and file data. - info, err = f.svc.Files.Insert(info).Media(in).Do() - } - if err != nil { - return nil, fmt.Errorf("Upload failed: %s", err) - } - fs.setMetaData(info) - return fs, nil + return fs, fs.Update(in, modTime, size) } // Mkdir creates the container if it doesn't exist @@ -888,6 +840,63 @@ func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) { return res.Body, nil } +// Update the object +// +// Copy the reader into the object updating modTime and size +// +// The new object may have been created if an error is returned +func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) error { + f := o.drive + directory, leaf := splitPath(o.remote) + directoryId, err := f.findDir(directory, true) + if err != nil { + return fmt.Errorf("Couldn't find or make directory: %s", err) + } + + // See if the file already exists + var info *drive.File + found, err := f.listAll(directoryId, leaf, false, true, func(item *drive.File) bool { + info = item + return true + }) + if err != nil { + return fmt.Errorf("Error finding file: %s", leaf, err) + } + + // Guess the mime type + mimeType := mime.TypeByExtension(path.Ext(o.remote)) + if mimeType == "" { + mimeType = "application/octet-stream" + } + modifiedDate := modTime.Format(time.RFC3339Nano) + + if found { + // Modify metadata + info.ModifiedDate = modifiedDate + info.MimeType = mimeType + + // Make the API request to upload metadata and file data. + info, err = f.svc.Files.Update(info.Id, info).SetModifiedDate(true).Media(in).Do() + } else { + // Define the metadata for the file we are going to create. + info = &drive.File{ + Title: leaf, + Description: leaf, + Parents: []*drive.ParentReference{{Id: directoryId}}, + MimeType: mimeType, + ModifiedDate: modifiedDate, + } + + // Make the API request to upload metadata and file data. + info, err = f.svc.Files.Insert(info).Media(in).Do() + } + if err != nil { + return fmt.Errorf("Upload failed: %s", err) + } + o.setMetaData(info) + return nil +} + // Remove an object func (o *FsObjectDrive) Remove() error { return o.drive.svc.Files.Delete(o.id).Do() diff --git a/fs/fs.go b/fs/fs.go index df5bcf08b..aa1e2df80 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -103,6 +103,9 @@ type Object interface { // Open opens the file for read. Call Close() on the returned io.ReadCloser Open() (io.ReadCloser, error) + // Update in to the object with the modTime given of the given size + Update(in io.Reader, modTime time.Time, size int64) error + // Storable says whether this object can be stored Storable() bool diff --git a/fs/operations.go b/fs/operations.go index 6fcd16d9c..d9408c1b6 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -97,8 +97,12 @@ func Equal(src, dst Object) bool { return true } -// Copy src object to f -func Copy(f Fs, src Object) { +// Copy src object to dst or f if nil +// +// If dst is nil then the object must not exist already. If you do +// call Copy() with dst nil on a pre-existing file then some filing +// systems (eg Drive) may duplicate the file. +func Copy(f Fs, dst, src Object) { in0, err := src.Open() if err != nil { Stats.Error() @@ -107,7 +111,11 @@ func Copy(f Fs, src Object) { } in := NewAccount(in0) // account the transfer - dst, err := f.Put(in, src.Remote(), src.ModTime(), src.Size()) + if dst != nil { + err = dst.Update(in, src.ModTime(), src.Size()) + } else { + dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size()) + } inErr := in.Close() if err == nil { err = inErr @@ -167,7 +175,7 @@ func Copier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) { for pair := range in { src := pair.src Stats.Transferring(src) - Copy(fdst, src) + Copy(fdst, pair.dst, src) Stats.DoneTransferring(src) } } diff --git a/local/local.go b/local/local.go index 9b13cf0dd..b6da7b36b 100644 --- a/local/local.go +++ b/local/local.go @@ -164,30 +164,7 @@ func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64 dstPath := filepath.Join(f.root, remote) // Temporary FsObject under construction fs := &FsObjectLocal{local: f, remote: remote, path: dstPath} - - dir := path.Dir(dstPath) - err := os.MkdirAll(dir, 0770) - if err != nil { - return fs, err - } - - out, err := os.Create(dstPath) - if err != nil { - return fs, err - } - - _, err = io.Copy(out, in) - outErr := out.Close() - if err != nil { - return fs, err - } - if outErr != nil { - return fs, outErr - } - - // Set the mtime - fs.SetModTime(modTime) - return fs, err + return fs, fs.Update(in, modTime, size) } // Mkdir creates the directory if it doesn't exist @@ -335,6 +312,33 @@ func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) { return } +// Update the object from in with modTime and size +func (o *FsObjectLocal) Update(in io.Reader, modTime time.Time, size int64) error { + dir := path.Dir(o.path) + err := os.MkdirAll(dir, 0770) + if err != nil { + return err + } + + out, err := os.Create(o.path) + if err != nil { + return err + } + + _, err = io.Copy(out, in) + outErr := out.Close() + if err != nil { + return err + } + if outErr != nil { + return outErr + } + + // Set the mtime + o.SetModTime(modTime) + return nil +} + // Stat a FsObject into info func (o *FsObjectLocal) lstat() error { info, err := os.Lstat(o.path) diff --git a/s3/s3.go b/s3/s3.go index b2c7f0c03..f37a03216 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -290,20 +290,7 @@ func (f *FsS3) ListDir() fs.DirChan { func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { // Temporary FsObject under construction fs := &FsObjectS3{s3: f, remote: remote} - - // Set the mtime in the headers - headers := s3.Headers{ - metaMtime: swift.TimeToFloatString(modTime), - } - - // Guess the content type - contentType := mime.TypeByExtension(path.Ext(remote)) - if contentType == "" { - contentType = "application/octet-stream" - } - - _, err := fs.s3.b.PutReaderHeaders(remote, in, size, contentType, f.perm, headers) - return fs, err + return fs, fs.Update(in, modTime, size) } // Mkdir creates the bucket if it doesn't exist @@ -438,6 +425,23 @@ func (o *FsObjectS3) Open() (in io.ReadCloser, err error) { return } +// Update the Object from in with modTime and size +func (o *FsObjectS3) Update(in io.Reader, modTime time.Time, size int64) error { + // Set the mtime in the headers + headers := s3.Headers{ + metaMtime: swift.TimeToFloatString(modTime), + } + + // Guess the content type + contentType := mime.TypeByExtension(path.Ext(o.remote)) + if contentType == "" { + contentType = "application/octet-stream" + } + + _, err := o.s3.b.PutReaderHeaders(o.remote, in, size, contentType, o.s3.perm, headers) + return err +} + // Remove an object func (o *FsObjectS3) Remove() error { return o.s3.b.Del(o.remote) diff --git a/swift/swift.go b/swift/swift.go index 65604ecfa..00a5e741d 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -210,20 +210,15 @@ func (f *FsSwift) ListDir() fs.DirChan { return out } -// Put the FsObject into the container +// Put the object into the container // // Copy the reader in to the new object which is returned // -// The new object may have been created +// The new object may have been created if an error is returned func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { // Temporary FsObject under construction fs := &FsObjectSwift{swift: f, remote: remote} - - // Set the mtime - m := swift.Metadata{} - m.SetModTime(modTime) - _, err := f.c.ObjectPut(f.container, remote, in, true, "", "", m.ObjectHeaders()) - return fs, err + return fs, fs.Update(in, modTime, size) } // Mkdir creates the container if it doesn't exist @@ -337,6 +332,17 @@ func (o *FsObjectSwift) Open() (in io.ReadCloser, err error) { return } +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *FsObjectSwift) Update(in io.Reader, modTime time.Time, size int64) error { + // Set the mtime + m := swift.Metadata{} + m.SetModTime(modTime) + _, err := o.swift.c.ObjectPut(o.swift.container, o.remote, in, true, "", "", m.ObjectHeaders()) + return err +} + // Remove an object func (o *FsObjectSwift) Remove() error { return o.swift.c.ObjectDelete(o.swift.container, o.remote)