From 1473de3f040af01c0d554cc6f6490c826bbbf3fa Mon Sep 17 00:00:00 2001 From: nielash Date: Thu, 22 Feb 2024 09:17:14 -0500 Subject: [PATCH] onedrive: add metadata support This change adds support for metadata on OneDrive. Metadata (including permissions) is supported for both files and directories. OneDrive supports System Metadata (not User Metadata, as of this writing.) Much of the metadata is read-only, and there are some differences between OneDrive Personal and Business (see table in OneDrive backend docs for details). Permissions are also supported, if --onedrive-metadata-permissions is set. The accepted values for --onedrive-metadata-permissions are read, write, read,write, and off (the default). write supports adding new permissions, updating the "role" of existing permissions, and removing permissions. Updating and removing require the Permission ID to be known, so it is recommended to use read,write instead of write if you wish to update/remove permissions. Permissions are read/written in JSON format using the same schema as the OneDrive API, which differs slightly between OneDrive Personal and Business. (See OneDrive backend docs for examples.) To write permissions, pass in a "permissions" metadata key using this same format. The --metadata-mapper tool can be very helpful for this. When adding permissions, an email address can be provided in the User.ID or DisplayName properties of grantedTo or grantedToIdentities. Alternatively, an ObjectID can be provided in User.ID. At least one valid recipient must be provided in order to add a permission for a user. Creating a Public Link is also supported, if Link.Scope is set to "anonymous". Note that adding a permission can fail if a conflicting permission already exists for the file/folder. To update an existing permission, include both the Permission ID and the new roles to be assigned. roles is the only property that can be changed. To remove permissions, pass in a blob containing only the permissions you wish to keep (which can be empty, to remove all.) Note that both reading and writing permissions requires extra API calls, so if you don't need to read or write permissions it is recommended to omit --onedrive- metadata-permissions. Metadata and permissions are supported for Folders (directories) as well as Files. Note that setting the mtime or btime on a Folder requires one extra API call on OneDrive Business only. OneDrive does not currently support User Metadata. When writing metadata, only writeable system properties will be written -- any read-only or unrecognized keys passed in will be ignored. TIP: to see the metadata and permissions for any file or folder, run: rclone lsjson remote:path --stat -M --onedrive-metadata-permissions read See the OneDrive backend docs for a table of all the supported metadata properties. --- backend/onedrive/api/types.go | 178 +++- backend/onedrive/metadata.go | 951 +++++++++++++++++++++ backend/onedrive/metadata.md | 147 ++++ backend/onedrive/onedrive.go | 196 ++++- backend/onedrive/onedrive_internal_test.go | 464 ++++++++++ docs/content/onedrive.md | 209 +++++ docs/content/overview.md | 2 +- fs/log.go | 15 + 8 files changed, 2093 insertions(+), 69 deletions(-) create mode 100644 backend/onedrive/metadata.go create mode 100644 backend/onedrive/metadata.md create mode 100644 backend/onedrive/onedrive_internal_test.go diff --git a/backend/onedrive/api/types.go b/backend/onedrive/api/types.go index c8e7a5fd1..5e4ab6624 100644 --- a/backend/onedrive/api/types.go +++ b/backend/onedrive/api/types.go @@ -7,7 +7,7 @@ import ( ) const ( - timeFormat = `"` + time.RFC3339 + `"` + timeFormat = `"` + "2006-01-02T15:04:05.999Z" + `"` // PackageTypeOneNote is the package type value for OneNote files PackageTypeOneNote = "oneNote" @@ -40,17 +40,17 @@ var _ error = (*Error)(nil) // Identity represents an identity of an actor. For example, and actor // can be a user, device, or application. type Identity struct { - DisplayName string `json:"displayName"` - ID string `json:"id"` + DisplayName string `json:"displayName,omitempty"` + ID string `json:"id,omitempty"` } // IdentitySet is a keyed collection of Identity objects. It is used // to represent a set of identities associated with various events for // an item, such as created by or last modified by. type IdentitySet struct { - User Identity `json:"user"` - Application Identity `json:"application"` - Device Identity `json:"device"` + User Identity `json:"user,omitempty"` + Application Identity `json:"application,omitempty"` + Device Identity `json:"device,omitempty"` } // Quota groups storage space quota-related information on OneDrive into a single structure. @@ -150,16 +150,15 @@ type FileFacet struct { // facet can be used to specify the last modified date or created date // of the item as it was on the local device. type FileSystemInfoFacet struct { - CreatedDateTime Timestamp `json:"createdDateTime"` // The UTC date and time the file was created on a client. - LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // The UTC date and time the file was last modified on a client. + CreatedDateTime Timestamp `json:"createdDateTime,omitempty"` // The UTC date and time the file was created on a client. + LastModifiedDateTime Timestamp `json:"lastModifiedDateTime,omitempty"` // The UTC date and time the file was last modified on a client. } // DeletedFacet indicates that the item on OneDrive has been // deleted. In this version of the API, the presence (non-null) of the // facet value indicates that the file was deleted. A null (or // missing) value indicates that the file is not deleted. -type DeletedFacet struct { -} +type DeletedFacet struct{} // PackageFacet indicates that a DriveItem is the top level item // in a "package" or a collection of items that should be treated as a collection instead of individual items. @@ -168,31 +167,141 @@ type PackageFacet struct { Type string `json:"type"` } +// SharedType indicates a DriveItem has been shared with others. The resource includes information about how the item is shared. +// If a Driveitem has a non-null shared facet, the item has been shared. +type SharedType struct { + Owner IdentitySet `json:"owner,omitempty"` // The identity of the owner of the shared item. Read-only. + Scope string `json:"scope,omitempty"` // Indicates the scope of how the item is shared: anonymous, organization, or users. Read-only. + SharedBy IdentitySet `json:"sharedBy,omitempty"` // The identity of the user who shared the item. Read-only. + SharedDateTime Timestamp `json:"sharedDateTime,omitempty"` // The UTC date and time when the item was shared. Read-only. +} + +// SharingInvitationType groups invitation-related data items into a single structure. +type SharingInvitationType struct { + Email string `json:"email,omitempty"` // The email address provided for the recipient of the sharing invitation. Read-only. + InvitedBy *IdentitySet `json:"invitedBy,omitempty"` // Provides information about who sent the invitation that created this permission, if that information is available. Read-only. + SignInRequired bool `json:"signInRequired,omitempty"` // If true the recipient of the invitation needs to sign in in order to access the shared item. Read-only. +} + +// SharingLinkType groups link-related data items into a single structure. +// If a Permission resource has a non-null sharingLink facet, the permission represents a sharing link (as opposed to permissions granted to a person or group). +type SharingLinkType struct { + Application *Identity `json:"application,omitempty"` // The app the link is associated with. + Type LinkType `json:"type,omitempty"` // The type of the link created. + Scope LinkScope `json:"scope,omitempty"` // The scope of the link represented by this permission. Value anonymous indicates the link is usable by anyone, organization indicates the link is only usable for users signed into the same tenant. + WebHTML string `json:"webHtml,omitempty"` // For embed links, this property contains the HTML code for an