diff --git a/backend/drive/metadata.go b/backend/drive/metadata.go
index 72a58c154..ee547eed0 100644
--- a/backend/drive/metadata.go
+++ b/backend/drive/metadata.go
@@ -9,6 +9,7 @@ import (
 	"sync"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/errcount"
 	"golang.org/x/sync/errgroup"
 	drive "google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
@@ -135,24 +136,26 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
 
 // Set the permissions on the info
 func (f *Fs) setPermissions(ctx context.Context, info *drive.File, permissions []*drive.Permission) (err error) {
+	errs := errcount.New()
 	for _, perm := range permissions {
 		if perm.Role == "owner" {
 			// ignore owner permissions - these are set with owner
 			continue
 		}
 		cleanPermissionForWrite(perm)
-		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Permissions.Create(info.Id, perm).
+		err := f.pacer.Call(func() (bool, error) {
+			_, err := f.svc.Permissions.Create(info.Id, perm).
 				SupportsAllDrives(true).
 				SendNotificationEmail(false).
 				Context(ctx).Do()
 			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
-			return fmt.Errorf("failed to set permission: %w", err)
+			fs.Errorf(f, "Failed to set permission: %v", err)
+			errs.Add(err)
 		}
 	}
-	return nil
+	return errs.Err("failed to set permission")
 }
 
 // Clean attributes from permissions which we can't write