drive: Use o.Update and fs.Put to optimise transfers

drive/drive.go

@@ -650,13 +650,46 @@ func (f *FsDrive) ListDir() fs.DirChan {
 
 // Put the object
 //
+// This assumes that the object doesn't not already exists - if you
+// call it when it does exist then it will create a duplicate.  Call
+// object.Update() in this case.
+//
 // Copy the reader in to the new object which is returned
 //
 // The new object may have been created if an error is returned
 func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
 	// Temporary FsObject under construction
-	fs := &FsObjectDrive{drive: f, remote: remote}
+	o := &FsObjectDrive{drive: f, remote: remote}
-	return fs, fs.Update(in, modTime, size)
+
+	directory, leaf := splitPath(o.remote)
+	directoryId, err := f.findDir(directory, true)
+	if err != nil {
+		return o, fmt.Errorf("Couldn't find or make directory: %s", err)
+	}
+
+	// Guess the mime type
+	mimeType := mime.TypeByExtension(path.Ext(o.remote))
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+	modifiedDate := modTime.Format(time.RFC3339Nano)
+
+	// Define the metadata for the file we are going to create.
+	info := &drive.File{
+		Title:        leaf,
+		Description:  leaf,
+		Parents:      []*drive.ParentReference{{Id: directoryId}},
+		MimeType:     mimeType,
+		ModifiedDate: modifiedDate,
+	}
+
+	// Make the API request to upload metadata and file data.
+	info, err = f.svc.Files.Insert(info).Media(in).Do()
+	if err != nil {
+		return o, fmt.Errorf("Upload failed: %s", err)
+	}
+	o.setMetaData(info)
+	return o, nil
 }
 
 // Mkdir creates the container if it doesn't exist
@@ -840,58 +873,21 @@ func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) {
 	return res.Body, nil
 }
 
-// Update the object
+// Update the already existing object
 //
 // Copy the reader into the object updating modTime and size
 //
 // The new object may have been created if an error is returned
 func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) error {
-	f := o.drive
+	info := &drive.File{
-	directory, leaf := splitPath(o.remote)
+		Id:           o.id,
-	directoryId, err := f.findDir(directory, true)
+		ModifiedDate: modTime.Format(time.RFC3339Nano),
+	}
+
+	// Make the API request to upload metadata and file data.
+	info, err := o.drive.svc.Files.Update(info.Id, info).SetModifiedDate(true).Media(in).Do()
 	if err != nil {
-		return fmt.Errorf("Couldn't find or make directory: %s", err)
+		return fmt.Errorf("Update failed: %s", err)
-	}
-
-	// See if the file already exists
-	var info *drive.File
-	found, err := f.listAll(directoryId, leaf, false, true, func(item *drive.File) bool {
-		info = item
-		return true
-	})
-	if err != nil {
-		return fmt.Errorf("Error finding file: %s", leaf, err)
-	}
-
-	// Guess the mime type
-	mimeType := mime.TypeByExtension(path.Ext(o.remote))
-	if mimeType == "" {
-		mimeType = "application/octet-stream"
-	}
-	modifiedDate := modTime.Format(time.RFC3339Nano)
-
-	if found {
-		// Modify metadata
-		info.ModifiedDate = modifiedDate
-		info.MimeType = mimeType
-
-		// Make the API request to upload metadata and file data.
-		info, err = f.svc.Files.Update(info.Id, info).SetModifiedDate(true).Media(in).Do()
-	} else {
-		// Define the metadata for the file we are going to create.
-		info = &drive.File{
-			Title:        leaf,
-			Description:  leaf,
-			Parents:      []*drive.ParentReference{{Id: directoryId}},
-			MimeType:     mimeType,
-			ModifiedDate: modifiedDate,
-		}
-
-		// Make the API request to upload metadata and file data.
-		info, err = f.svc.Files.Insert(info).Media(in).Do()
-	}
-	if err != nil {
-		return fmt.Errorf("Upload failed: %s", err)
 	}
 	o.setMetaData(info)
 	return nil

fs/operations.go

@@ -111,9 +111,12 @@ func Copy(f Fs, dst, src Object) {
 	}
 	in := NewAccount(in0) // account the transfer
 
+	var actionTaken string
 	if dst != nil {
+		actionTaken = "Copied (updated existing)"
 		err = dst.Update(in, src.ModTime(), src.Size())
 	} else {
+		actionTaken = "Copied (new)"
 		dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size())
 	}
 	inErr := in.Close()
@@ -133,7 +136,7 @@ func Copy(f Fs, dst, src Object) {
 		}
 		return
 	}
-	Debug(src, "Copied")
+	Debug(src, actionTaken)
 }
 
 // Check to see if src needs to be copied to dst and if so puts it in out

notes.txt

@@ -1,6 +1,4 @@
 Todo
-  * Make a test suite which can run on all the given types of fs
-  * Copy should use the sync code as it is more efficient at directory listing
   * FIXME: ls without an argument for buckets/containers?
   * FIXME: More -dry-run checks for object transfer
   * Might be quicker to check md5sums first? for swift <-> swift certainly, and maybe for small files
@@ -43,10 +41,5 @@ s3
     * Otherwise can set metadata
   * Returns etag and last modified in bucket list
 
-Drive
-  * Should keep a note of files we list then call a new method
-    Object.Update() which would be more efficient than haveing to look
-    the id up for each file
- 
 Bugs
   * Non verbose - not sure number transferred got counted up? CHECK