2023-09-14 15:32:35 +00:00
package drive
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"sync"
"github.com/rclone/rclone/fs"
2024-04-03 15:50:26 +00:00
"github.com/rclone/rclone/lib/errcount"
2023-09-14 15:32:35 +00:00
"golang.org/x/sync/errgroup"
drive "google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
)
// system metadata keys which this backend owns
var systemMetadataInfo = map [ string ] fs . MetadataHelp {
"content-type" : {
Help : "The MIME type of the file." ,
Type : "string" ,
Example : "text/plain" ,
} ,
"mtime" : {
Help : "Time of last modification with mS accuracy." ,
Type : "RFC 3339" ,
Example : "2006-01-02T15:04:05.999Z07:00" ,
} ,
"btime" : {
2023-10-24 16:39:33 +00:00
Help : "Time of file birth (creation) with mS accuracy. Note that this is only writable on fresh uploads - it can't be written for updates." ,
2023-09-14 15:32:35 +00:00
Type : "RFC 3339" ,
Example : "2006-01-02T15:04:05.999Z07:00" ,
} ,
"copy-requires-writer-permission" : {
Help : "Whether the options to copy, print, or download this file, should be disabled for readers and commenters." ,
Type : "boolean" ,
Example : "true" ,
} ,
"writers-can-share" : {
2024-03-14 17:54:55 +00:00
Help : "Whether users with only writer permission can modify the file's permissions. Not populated and ignored when setting for items in shared drives." ,
2023-09-14 15:32:35 +00:00
Type : "boolean" ,
Example : "false" ,
} ,
"viewed-by-me" : {
Help : "Whether the file has been viewed by this user." ,
Type : "boolean" ,
Example : "true" ,
ReadOnly : true ,
} ,
"owner" : {
Help : "The owner of the file. Usually an email address. Enable with --drive-metadata-owner." ,
Type : "string" ,
Example : "user@example.com" ,
} ,
"permissions" : {
Help : "Permissions in a JSON dump of Google drive format. On shared drives these will only be present if they aren't inherited. Enable with --drive-metadata-permissions." ,
Type : "JSON" ,
Example : "{}" ,
} ,
"folder-color-rgb" : {
Help : "The color for a folder or a shortcut to a folder as an RGB hex string." ,
Type : "string" ,
Example : "881133" ,
} ,
"description" : {
Help : "A short description of the file." ,
Type : "string" ,
Example : "Contract for signing" ,
} ,
"starred" : {
Help : "Whether the user has starred the file." ,
Type : "boolean" ,
Example : "false" ,
} ,
"labels" : {
Help : "Labels attached to this file in a JSON dump of Googled drive format. Enable with --drive-metadata-labels." ,
Type : "JSON" ,
Example : "[]" ,
} ,
}
// Extra fields we need to fetch to implement the system metadata above
var metadataFields = googleapi . Field ( strings . Join ( [ ] string {
"copyRequiresWriterPermission" ,
"description" ,
"folderColorRgb" ,
"hasAugmentedPermissions" ,
"owners" ,
"permissionIds" ,
"permissions" ,
"properties" ,
"starred" ,
"viewedByMe" ,
"viewedByMeTime" ,
"writersCanShare" ,
} , "," ) )
// Fields we need to read from permissions
var permissionsFields = googleapi . Field ( strings . Join ( [ ] string {
"*" ,
"permissionDetails/*" ,
} , "," ) )
// getPermission returns permissions for the fileID and permissionID passed in
func ( f * Fs ) getPermission ( ctx context . Context , fileID , permissionID string , useCache bool ) ( perm * drive . Permission , inherited bool , err error ) {
f . permissionsMu . Lock ( )
defer f . permissionsMu . Unlock ( )
if useCache {
perm = f . permissions [ permissionID ]
if perm != nil {
return perm , false , nil
}
}
fs . Debugf ( f , "Fetching permission %q" , permissionID )
err = f . pacer . Call ( func ( ) ( bool , error ) {
perm , err = f . svc . Permissions . Get ( fileID , permissionID ) .
Fields ( permissionsFields ) .
SupportsAllDrives ( true ) .
Context ( ctx ) . Do ( )
return f . shouldRetry ( ctx , err )
} )
if err != nil {
return nil , false , err
}
inherited = len ( perm . PermissionDetails ) > 0 && perm . PermissionDetails [ 0 ] . Inherited
cleanPermission ( perm )
// cache the permission
f . permissions [ permissionID ] = perm
return perm , inherited , err
}
// Set the permissions on the info
func ( f * Fs ) setPermissions ( ctx context . Context , info * drive . File , permissions [ ] * drive . Permission ) ( err error ) {
2024-04-03 15:50:26 +00:00
errs := errcount . New ( )
2023-09-14 15:32:35 +00:00
for _ , perm := range permissions {
if perm . Role == "owner" {
// ignore owner permissions - these are set with owner
continue
}
cleanPermissionForWrite ( perm )
2024-04-03 15:50:26 +00:00
err := f . pacer . Call ( func ( ) ( bool , error ) {
_ , err := f . svc . Permissions . Create ( info . Id , perm ) .
2023-09-14 15:32:35 +00:00
SupportsAllDrives ( true ) .
2024-03-25 17:23:57 +00:00
SendNotificationEmail ( false ) .
2023-09-14 15:32:35 +00:00
Context ( ctx ) . Do ( )
return f . shouldRetry ( ctx , err )
} )
if err != nil {
2024-04-03 15:50:26 +00:00
fs . Errorf ( f , "Failed to set permission: %v" , err )
errs . Add ( err )
2023-09-14 15:32:35 +00:00
}
}
2024-04-03 15:50:26 +00:00
return errs . Err ( "failed to set permission" )
2023-09-14 15:32:35 +00:00
}
// Clean attributes from permissions which we can't write
func cleanPermissionForWrite ( perm * drive . Permission ) {
perm . Deleted = false
perm . DisplayName = ""
perm . Id = ""
perm . Kind = ""
perm . PermissionDetails = nil
perm . TeamDrivePermissionDetails = nil
}
// Clean and cache the permission if not already cached
func ( f * Fs ) cleanAndCachePermission ( perm * drive . Permission ) {
f . permissionsMu . Lock ( )
defer f . permissionsMu . Unlock ( )
cleanPermission ( perm )
if _ , found := f . permissions [ perm . Id ] ; ! found {
f . permissions [ perm . Id ] = perm
}
}
// Clean fields we don't need to keep from the permission
func cleanPermission ( perm * drive . Permission ) {
// DisplayName: Output only. The "pretty" name of the value of the
// permission. The following is a list of examples for each type of
// permission: * `user` - User's full name, as defined for their Google
// account, such as "Joe Smith." * `group` - Name of the Google Group,
// such as "The Company Administrators." * `domain` - String domain
// name, such as "thecompany.com." * `anyone` - No `displayName` is
// present.
perm . DisplayName = ""
// Kind: Output only. Identifies what kind of resource this is. Value:
// the fixed string "drive#permission".
perm . Kind = ""
// PermissionDetails: Output only. Details of whether the permissions on
// this shared drive item are inherited or directly on this item. This
// is an output-only field which is present only for shared drive items.
perm . PermissionDetails = nil
// PhotoLink: Output only. A link to the user's profile photo, if
// available.
perm . PhotoLink = ""
// TeamDrivePermissionDetails: Output only. Deprecated: Output only. Use
// `permissionDetails` instead.
perm . TeamDrivePermissionDetails = nil
}
// Fields we need to read from labels
var labelsFields = googleapi . Field ( strings . Join ( [ ] string {
"*" ,
} , "," ) )
// getLabels returns labels for the fileID passed in
func ( f * Fs ) getLabels ( ctx context . Context , fileID string ) ( labels [ ] * drive . Label , err error ) {
fs . Debugf ( f , "Fetching labels for %q" , fileID )
listLabels := f . svc . Files . ListLabels ( fileID ) .
Fields ( labelsFields ) .
Context ( ctx )
for {
var info * drive . LabelList
err = f . pacer . Call ( func ( ) ( bool , error ) {
info , err = listLabels . Do ( )
return f . shouldRetry ( ctx , err )
} )
if err != nil {
return nil , err
}
labels = append ( labels , info . Labels ... )
if info . NextPageToken == "" {
break
}
listLabels . PageToken ( info . NextPageToken )
}
for _ , label := range labels {
cleanLabel ( label )
}
return labels , nil
}
// Set the labels on the info
func ( f * Fs ) setLabels ( ctx context . Context , info * drive . File , labels [ ] * drive . Label ) ( err error ) {
if len ( labels ) == 0 {
return nil
}
req := drive . ModifyLabelsRequest { }
for _ , label := range labels {
req . LabelModifications = append ( req . LabelModifications , & drive . LabelModification {
FieldModifications : labelFieldsToFieldModifications ( label . Fields ) ,
LabelId : label . Id ,
} )
}
err = f . pacer . Call ( func ( ) ( bool , error ) {
_ , err = f . svc . Files . ModifyLabels ( info . Id , & req ) .
Context ( ctx ) . Do ( )
return f . shouldRetry ( ctx , err )
} )
if err != nil {
return fmt . Errorf ( "failed to set owner: %w" , err )
}
return nil
}
// Convert label fields into something which can set the fields
func labelFieldsToFieldModifications ( fields map [ string ] drive . LabelField ) ( out [ ] * drive . LabelFieldModification ) {
for id , field := range fields {
var emails [ ] string
for _ , user := range field . User {
emails = append ( emails , user . EmailAddress )
}
out = append ( out , & drive . LabelFieldModification {
// FieldId: The ID of the field to be modified.
FieldId : id ,
// SetDateValues: Replaces the value of a dateString Field with these
// new values. The string must be in the RFC 3339 full-date format:
// YYYY-MM-DD.
SetDateValues : field . DateString ,
// SetIntegerValues: Replaces the value of an `integer` field with these
// new values.
SetIntegerValues : field . Integer ,
// SetSelectionValues: Replaces a `selection` field with these new
// values.
SetSelectionValues : field . Selection ,
// SetTextValues: Sets the value of a `text` field.
SetTextValues : field . Text ,
// SetUserValues: Replaces a `user` field with these new values. The
// values must be valid email addresses.
SetUserValues : emails ,
} )
}
return out
}
// Clean fields we don't need to keep from the label
func cleanLabel ( label * drive . Label ) {
// Kind: This is always drive#label
label . Kind = ""
for name , field := range label . Fields {
// Kind: This is always drive#labelField.
field . Kind = ""
// Note the fields are copies so we need to write them
// back to the map
label . Fields [ name ] = field
}
}
// Parse the metadata from drive item
//
// It should return nil if there is no Metadata
func ( o * baseObject ) parseMetadata ( ctx context . Context , info * drive . File ) ( err error ) {
metadata := make ( fs . Metadata , 16 )
// Dump user metadata first as it overrides system metadata
for k , v := range info . Properties {
metadata [ k ] = v
}
// System metadata
metadata [ "copy-requires-writer-permission" ] = fmt . Sprint ( info . CopyRequiresWriterPermission )
metadata [ "writers-can-share" ] = fmt . Sprint ( info . WritersCanShare )
metadata [ "viewed-by-me" ] = fmt . Sprint ( info . ViewedByMe )
metadata [ "content-type" ] = info . MimeType
// Owners: Output only. The owner of this file. Only certain legacy
// files may have more than one owner. This field isn't populated for
// items in shared drives.
if o . fs . opt . MetadataOwner . IsSet ( rwRead ) && len ( info . Owners ) > 0 {
user := info . Owners [ 0 ]
if len ( info . Owners ) > 1 {
fs . Logf ( o , "Ignoring more than 1 owner" )
}
if user != nil {
id := user . EmailAddress
if id == "" {
id = user . DisplayName
}
metadata [ "owner" ] = id
}
}
if o . fs . opt . MetadataPermissions . IsSet ( rwRead ) {
// We only write permissions out if they are not inherited.
//
// On My Drives permissions seem to be attached to every item
// so they will always be written out.
//
// On Shared Drives only non-inherited permissions will be
// written out.
// To read the inherited permissions flag will mean we need to
// read the permissions for each object and the cache will be
// useless. However shared drives don't return permissions
// only permissionIds so will need to fetch them for each
// object. We use HasAugmentedPermissions to see if there are
// special permissions before fetching them to save transactions.
// HasAugmentedPermissions: Output only. Whether there are permissions
// directly on this file. This field is only populated for items in
// shared drives.
if o . fs . isTeamDrive && ! info . HasAugmentedPermissions {
// Don't process permissions if there aren't any specifically set
info . Permissions = nil
info . PermissionIds = nil
}
// PermissionIds: Output only. List of permission IDs for users with
// access to this file.
//
// Only process these if we have no Permissions
if len ( info . PermissionIds ) > 0 && len ( info . Permissions ) == 0 {
info . Permissions = make ( [ ] * drive . Permission , 0 , len ( info . PermissionIds ) )
g , gCtx := errgroup . WithContext ( ctx )
g . SetLimit ( o . fs . ci . Checkers )
var mu sync . Mutex // protect the info.Permissions from concurrent writes
for _ , permissionID := range info . PermissionIds {
permissionID := permissionID
g . Go ( func ( ) error {
// must fetch the team drive ones individually to check the inherited flag
perm , inherited , err := o . fs . getPermission ( gCtx , actualID ( info . Id ) , permissionID , ! o . fs . isTeamDrive )
if err != nil {
return fmt . Errorf ( "failed to read permission: %w" , err )
}
// Don't write inherited permissions out
if inherited {
return nil
}
// Don't write owner role out - these are covered by the owner metadata
if perm . Role == "owner" {
return nil
}
mu . Lock ( )
info . Permissions = append ( info . Permissions , perm )
mu . Unlock ( )
return nil
} )
}
err = g . Wait ( )
if err != nil {
return err
}
} else {
// Clean the fetched permissions
for _ , perm := range info . Permissions {
o . fs . cleanAndCachePermission ( perm )
}
}
// Permissions: Output only. The full list of permissions for the file.
// This is only available if the requesting user can share the file. Not
// populated for items in shared drives.
if len ( info . Permissions ) > 0 {
buf , err := json . Marshal ( info . Permissions )
if err != nil {
return fmt . Errorf ( "failed to marshal permissions: %w" , err )
}
metadata [ "permissions" ] = string ( buf )
}
// Permission propagation
// https://developers.google.com/drive/api/guides/manage-sharing#permission-propagation
// Leads me to believe that in non shared drives, permissions
// are added to each item when you set permissions for a
// folder whereas in shared drives they are inherited and
// placed on the item directly.
}
if info . FolderColorRgb != "" {
metadata [ "folder-color-rgb" ] = info . FolderColorRgb
}
if info . Description != "" {
metadata [ "description" ] = info . Description
}
metadata [ "starred" ] = fmt . Sprint ( info . Starred )
metadata [ "btime" ] = info . CreatedTime
metadata [ "mtime" ] = info . ModifiedTime
if o . fs . opt . MetadataLabels . IsSet ( rwRead ) {
// FIXME would be really nice if we knew if files had labels
// before listing but we need to know all possible label IDs
// to get it in the listing.
labels , err := o . fs . getLabels ( ctx , actualID ( info . Id ) )
if err != nil {
return fmt . Errorf ( "failed to fetch labels: %w" , err )
}
buf , err := json . Marshal ( labels )
if err != nil {
return fmt . Errorf ( "failed to marshal labels: %w" , err )
}
metadata [ "labels" ] = string ( buf )
}
o . metadata = & metadata
return nil
}
// Set the owner on the info
func ( f * Fs ) setOwner ( ctx context . Context , info * drive . File , owner string ) ( err error ) {
perm := drive . Permission {
Role : "owner" ,
EmailAddress : owner ,
// Type: The type of the grantee. Valid values are: * `user` * `group` *
// `domain` * `anyone` When creating a permission, if `type` is `user`
// or `group`, you must provide an `emailAddress` for the user or group.
// When `type` is `domain`, you must provide a `domain`. There isn't
// extra information required for an `anyone` type.
Type : "user" ,
}
err = f . pacer . Call ( func ( ) ( bool , error ) {
_ , err = f . svc . Permissions . Create ( info . Id , & perm ) .
SupportsAllDrives ( true ) .
TransferOwnership ( true ) .
// SendNotificationEmail(false). - required apparently!
Context ( ctx ) . Do ( )
return f . shouldRetry ( ctx , err )
} )
if err != nil {
return fmt . Errorf ( "failed to set owner: %w" , err )
}
return nil
}
// Call back to set metadata that can't be set on the upload/update
//
// The *drive.File passed in holds the current state of the drive.File
// and this should update it with any modifications.
type updateMetadataFn func ( context . Context , * drive . File ) error
// read the metadata from meta and write it into updateInfo
//
2023-10-24 16:39:33 +00:00
// update should be true if this is being used to create metadata for
// an update/PATCH call as the rules on what can be updated are
// slightly different there.
//
2023-09-14 15:32:35 +00:00
// It returns a callback which should be called to finish the updates
// after the data is uploaded.
2023-10-24 16:39:33 +00:00
func ( f * Fs ) updateMetadata ( ctx context . Context , updateInfo * drive . File , meta fs . Metadata , update bool ) ( callback updateMetadataFn , err error ) {
2023-09-14 15:32:35 +00:00
callbackFns := [ ] updateMetadataFn { }
callback = func ( ctx context . Context , info * drive . File ) error {
for _ , fn := range callbackFns {
err := fn ( ctx , info )
if err != nil {
return err
}
}
return nil
}
// merge metadata into request and user metadata
for k , v := range meta {
k , v := k , v
// parse a boolean from v and write into out
parseBool := func ( out * bool ) error {
b , err := strconv . ParseBool ( v )
if err != nil {
return fmt . Errorf ( "can't parse metadata %q = %q: %w" , k , v , err )
}
* out = b
return nil
}
switch k {
case "copy-requires-writer-permission" :
if err := parseBool ( & updateInfo . CopyRequiresWriterPermission ) ; err != nil {
return nil , err
}
case "writers-can-share" :
2024-03-14 17:54:55 +00:00
if ! f . isTeamDrive {
if err := parseBool ( & updateInfo . WritersCanShare ) ; err != nil {
return nil , err
}
} else {
fs . Debugf ( f , "Ignoring %s=%s as can't set on shared drives" , k , v )
2023-09-14 15:32:35 +00:00
}
case "viewed-by-me" :
// Can't write this
case "content-type" :
updateInfo . MimeType = v
case "owner" :
if ! f . opt . MetadataOwner . IsSet ( rwWrite ) {
continue
}
// Can't set Owner on upload so need to set afterwards
callbackFns = append ( callbackFns , func ( ctx context . Context , info * drive . File ) error {
return f . setOwner ( ctx , info , v )
} )
case "permissions" :
if ! f . opt . MetadataPermissions . IsSet ( rwWrite ) {
continue
}
var perms [ ] * drive . Permission
err := json . Unmarshal ( [ ] byte ( v ) , & perms )
if err != nil {
return nil , fmt . Errorf ( "failed to unmarshal permissions: %w" , err )
}
// Can't set Permissions on upload so need to set afterwards
callbackFns = append ( callbackFns , func ( ctx context . Context , info * drive . File ) error {
return f . setPermissions ( ctx , info , perms )
} )
case "labels" :
if ! f . opt . MetadataLabels . IsSet ( rwWrite ) {
continue
}
var labels [ ] * drive . Label
err := json . Unmarshal ( [ ] byte ( v ) , & labels )
if err != nil {
return nil , fmt . Errorf ( "failed to unmarshal labels: %w" , err )
}
// Can't set Labels on upload so need to set afterwards
callbackFns = append ( callbackFns , func ( ctx context . Context , info * drive . File ) error {
return f . setLabels ( ctx , info , labels )
} )
case "folder-color-rgb" :
updateInfo . FolderColorRgb = v
case "description" :
updateInfo . Description = v
case "starred" :
if err := parseBool ( & updateInfo . Starred ) ; err != nil {
return nil , err
}
case "btime" :
2023-10-24 16:39:33 +00:00
if update {
fs . Debugf ( f , "Skipping btime metadata as can't update it on an existing file: %v" , v )
} else {
updateInfo . CreatedTime = v
}
2023-09-14 15:32:35 +00:00
case "mtime" :
updateInfo . ModifiedTime = v
default :
if updateInfo . Properties == nil {
updateInfo . Properties = make ( map [ string ] string , 1 )
}
updateInfo . Properties [ k ] = v
}
}
return callback , nil
}
// Fetch metadata and update updateInfo if --metadata is in use
2023-10-24 16:39:33 +00:00
func ( f * Fs ) fetchAndUpdateMetadata ( ctx context . Context , src fs . ObjectInfo , options [ ] fs . OpenOption , updateInfo * drive . File , update bool ) ( callback updateMetadataFn , err error ) {
2023-10-23 22:47:18 +00:00
meta , err := fs . GetMetadataOptions ( ctx , f , src , options )
2023-09-14 15:32:35 +00:00
if err != nil {
return nil , fmt . Errorf ( "failed to read metadata from source object: %w" , err )
}
2023-10-24 16:39:33 +00:00
callback , err = f . updateMetadata ( ctx , updateInfo , meta , update )
2023-09-14 15:32:35 +00:00
if err != nil {
return nil , fmt . Errorf ( "failed to update metadata from source object: %w" , err )
}
return callback , nil
}