690a44e40e
Make use of the mime package to find matching extensions and mime types. For simplicity, all extensions are now prefixed with "." to match the mime package requirements. Parsed extensions get converted if needed.
2257 lines
63 KiB
Go
2257 lines
63 KiB
Go
// Package drive interfaces with the Google Drive object storage system
|
|
package drive
|
|
|
|
// FIXME need to deal with some corner cases
|
|
// * multiple files with the same name
|
|
// * files can be in multiple directories
|
|
// * can have directory loops
|
|
// * files with / in name
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"log"
|
|
"mime"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/ncw/rclone/fs"
|
|
"github.com/ncw/rclone/fs/config"
|
|
"github.com/ncw/rclone/fs/config/configmap"
|
|
"github.com/ncw/rclone/fs/config/configstruct"
|
|
"github.com/ncw/rclone/fs/config/obscure"
|
|
"github.com/ncw/rclone/fs/fserrors"
|
|
"github.com/ncw/rclone/fs/fshttp"
|
|
"github.com/ncw/rclone/fs/hash"
|
|
"github.com/ncw/rclone/fs/walk"
|
|
"github.com/ncw/rclone/lib/dircache"
|
|
"github.com/ncw/rclone/lib/oauthutil"
|
|
"github.com/ncw/rclone/lib/pacer"
|
|
"github.com/pkg/errors"
|
|
"golang.org/x/oauth2"
|
|
"golang.org/x/oauth2/google"
|
|
drive_v2 "google.golang.org/api/drive/v2"
|
|
drive "google.golang.org/api/drive/v3"
|
|
"google.golang.org/api/googleapi"
|
|
)
|
|
|
|
// Constants
|
|
const (
|
|
rcloneClientID = "202264815644.apps.googleusercontent.com"
|
|
rcloneEncryptedClientSecret = "eX8GpZTVx3vxMWVkuuBdDWmAUE6rGhTwVrvG9GhllYccSdj2-mvHVg"
|
|
driveFolderType = "application/vnd.google-apps.folder"
|
|
timeFormatIn = time.RFC3339
|
|
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
|
|
minSleep = 10 * time.Millisecond
|
|
defaultExtensions = "docx,xlsx,pptx,svg"
|
|
scopePrefix = "https://www.googleapis.com/auth/"
|
|
defaultScope = "drive"
|
|
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
|
|
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
|
defaultChunkSize = fs.SizeSuffix(8 * 1024 * 1024)
|
|
)
|
|
|
|
// Globals
|
|
var (
|
|
// Description of how to auth for this app
|
|
driveConfig = &oauth2.Config{
|
|
Scopes: []string{scopePrefix + "drive"},
|
|
Endpoint: google.Endpoint,
|
|
ClientID: rcloneClientID,
|
|
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
|
|
RedirectURL: oauthutil.TitleBarRedirectURL,
|
|
}
|
|
_mimeTypeToExtensionDuplicates = map[string]string{
|
|
"application/x-vnd.oasis.opendocument.presentation": ".odp",
|
|
"application/x-vnd.oasis.opendocument.spreadsheet": ".ods",
|
|
"application/x-vnd.oasis.opendocument.text": ".odt",
|
|
"image/jpg": ".jpg",
|
|
"image/x-bmp": ".bmp",
|
|
"image/x-png": ".png",
|
|
"text/rtf": ".rtf",
|
|
}
|
|
_mimeTypeToExtension = map[string]string{
|
|
"application/epub+zip": ".epub",
|
|
"application/json": ".json",
|
|
"application/msword": ".doc",
|
|
"application/pdf": ".pdf",
|
|
"application/rtf": ".rtf",
|
|
"application/vnd.ms-excel": ".xls",
|
|
"application/vnd.oasis.opendocument.presentation": ".odp",
|
|
"application/vnd.oasis.opendocument.spreadsheet": ".ods",
|
|
"application/vnd.oasis.opendocument.text": ".odt",
|
|
"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
|
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
|
|
"application/x-msmetafile": ".wmf",
|
|
"application/zip": ".zip",
|
|
"image/bmp": ".bmp",
|
|
"image/jpeg": ".jpg",
|
|
"image/pjpeg": ".pjpeg",
|
|
"image/png": ".png",
|
|
"image/svg+xml": ".svg",
|
|
"text/csv": ".csv",
|
|
"text/html": ".html",
|
|
"text/plain": ".txt",
|
|
"text/tab-separated-values": ".tsv",
|
|
}
|
|
extensionToMimeType map[string]string
|
|
partialFields = "id,name,size,md5Checksum,trashed,modifiedTime,createdTime,mimeType,parents"
|
|
exportFormatsOnce sync.Once // make sure we fetch the export formats only once
|
|
_exportFormats map[string][]string // allowed export MIME type conversions
|
|
)
|
|
|
|
// Register with Fs
|
|
func init() {
|
|
fs.Register(&fs.RegInfo{
|
|
Name: "drive",
|
|
Description: "Google Drive",
|
|
NewFs: NewFs,
|
|
Config: func(name string, m configmap.Mapper) {
|
|
// Parse config into Options struct
|
|
opt := new(Options)
|
|
err := configstruct.Set(m, opt)
|
|
if err != nil {
|
|
fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
|
|
return
|
|
}
|
|
// Fill in the scopes
|
|
if opt.Scope == "" {
|
|
opt.Scope = defaultScope
|
|
}
|
|
driveConfig.Scopes = nil
|
|
for _, scope := range strings.Split(opt.Scope, ",") {
|
|
driveConfig.Scopes = append(driveConfig.Scopes, scopePrefix+strings.TrimSpace(scope))
|
|
// Set the root_folder_id if using drive.appfolder
|
|
if scope == "drive.appfolder" {
|
|
m.Set("root_folder_id", "appDataFolder")
|
|
}
|
|
}
|
|
if opt.ServiceAccountFile == "" {
|
|
err = oauthutil.Config("drive", name, m, driveConfig)
|
|
if err != nil {
|
|
log.Fatalf("Failed to configure token: %v", err)
|
|
}
|
|
}
|
|
err = configTeamDrive(opt, m, name)
|
|
if err != nil {
|
|
log.Fatalf("Failed to configure team drive: %v", err)
|
|
}
|
|
},
|
|
Options: []fs.Option{{
|
|
Name: config.ConfigClientID,
|
|
Help: "Google Application Client Id\nLeave blank normally.",
|
|
}, {
|
|
Name: config.ConfigClientSecret,
|
|
Help: "Google Application Client Secret\nLeave blank normally.",
|
|
}, {
|
|
Name: "scope",
|
|
Help: "Scope that rclone should use when requesting access from drive.",
|
|
Examples: []fs.OptionExample{{
|
|
Value: "drive",
|
|
Help: "Full access all files, excluding Application Data Folder.",
|
|
}, {
|
|
Value: "drive.readonly",
|
|
Help: "Read-only access to file metadata and file contents.",
|
|
}, {
|
|
Value: "drive.file",
|
|
Help: "Access to files created by rclone only.\nThese are visible in the drive website.\nFile authorization is revoked when the user deauthorizes the app.",
|
|
}, {
|
|
Value: "drive.appfolder",
|
|
Help: "Allows read and write access to the Application Data folder.\nThis is not visible in the drive website.",
|
|
}, {
|
|
Value: "drive.metadata.readonly",
|
|
Help: "Allows read-only access to file metadata but\ndoes not allow any access to read or download file content.",
|
|
}},
|
|
}, {
|
|
Name: "root_folder_id",
|
|
Help: "ID of the root folder\nLeave blank normally.\nFill in to access \"Computers\" folders. (see docs).",
|
|
}, {
|
|
Name: "service_account_file",
|
|
Help: "Service Account Credentials JSON file path \nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
|
}, {
|
|
Name: "service_account_credentials",
|
|
Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
|
Hide: fs.OptionHideBoth,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "team_drive",
|
|
Help: "ID of the Team Drive",
|
|
Hide: fs.OptionHideBoth,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "auth_owner_only",
|
|
Default: false,
|
|
Help: "Only consider files owned by the authenticated user.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "use_trash",
|
|
Default: true,
|
|
Help: "Send files to the trash instead of deleting permanently.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "skip_gdocs",
|
|
Default: false,
|
|
Help: "Skip google documents in all listings.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "shared_with_me",
|
|
Default: false,
|
|
Help: "Only show files that are shared with me",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "trashed_only",
|
|
Default: false,
|
|
Help: "Only show files that are in the trash",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "formats",
|
|
Default: defaultExtensions,
|
|
Help: "Comma separated list of preferred formats for downloading Google docs.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "use_created_date",
|
|
Default: false,
|
|
Help: "Use created date instead of modified date.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "list_chunk",
|
|
Default: 1000,
|
|
Help: "Size of listing chunk 100-1000. 0 to disable.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "impersonate",
|
|
Default: "",
|
|
Help: "Impersonate this user when using a service account.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "alternate_export",
|
|
Default: false,
|
|
Help: "Use alternate export URLs for google documents export.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "upload_cutoff",
|
|
Default: defaultChunkSize,
|
|
Help: "Cutoff for switching to chunked upload",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "chunk_size",
|
|
Default: defaultChunkSize,
|
|
Help: "Upload chunk size. Must a power of 2 >= 256k.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "acknowledge_abuse",
|
|
Default: false,
|
|
Help: "Set to allow files which return cannotDownloadAbusiveFile to be downloaded.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "keep_revision_forever",
|
|
Default: false,
|
|
Help: "Keep new head revision forever.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "v2_download_min_size",
|
|
Default: fs.SizeSuffix(-1),
|
|
Help: "If Object's are greater, use drive v2 API to download.",
|
|
Advanced: true,
|
|
}},
|
|
})
|
|
|
|
// register duplicate MIME types first
|
|
// this allows them to be used with mime.ExtensionsByType() but
|
|
// mime.TypeByExtension() will return the later registered MIME type
|
|
for _, m := range []map[string]string{_mimeTypeToExtensionDuplicates, _mimeTypeToExtension} {
|
|
for mimeType, extension := range m {
|
|
if err := mime.AddExtensionType(extension, mimeType); err != nil {
|
|
log.Fatalf("Failed to register MIME type %q: %v", mimeType, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Options defines the configuration for this backend
|
|
type Options struct {
|
|
Scope string `config:"scope"`
|
|
RootFolderID string `config:"root_folder_id"`
|
|
ServiceAccountFile string `config:"service_account_file"`
|
|
ServiceAccountCredentials string `config:"service_account_credentials"`
|
|
TeamDriveID string `config:"team_drive"`
|
|
AuthOwnerOnly bool `config:"auth_owner_only"`
|
|
UseTrash bool `config:"use_trash"`
|
|
SkipGdocs bool `config:"skip_gdocs"`
|
|
SharedWithMe bool `config:"shared_with_me"`
|
|
TrashedOnly bool `config:"trashed_only"`
|
|
Extensions string `config:"formats"`
|
|
UseCreatedDate bool `config:"use_created_date"`
|
|
ListChunk int64 `config:"list_chunk"`
|
|
Impersonate string `config:"impersonate"`
|
|
AlternateExport bool `config:"alternate_export"`
|
|
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
|
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
|
AcknowledgeAbuse bool `config:"acknowledge_abuse"`
|
|
KeepRevisionForever bool `config:"keep_revision_forever"`
|
|
V2DownloadMinSize fs.SizeSuffix `config:"v2_download_min_size"`
|
|
}
|
|
|
|
// Fs represents a remote drive server
|
|
type Fs struct {
|
|
name string // name of this remote
|
|
root string // the path we are working on
|
|
opt Options // parsed options
|
|
features *fs.Features // optional features
|
|
svc *drive.Service // the connection to the drive server
|
|
v2Svc *drive_v2.Service // used to create download links for the v2 api
|
|
client *http.Client // authorized client
|
|
rootFolderID string // the id of the root folder
|
|
dirCache *dircache.DirCache // Map of directory path to directory id
|
|
pacer *pacer.Pacer // To pace the API calls
|
|
extensions []string // preferred extensions to download docs
|
|
isTeamDrive bool // true if this is a team drive
|
|
}
|
|
|
|
// Object describes a drive object
|
|
type Object struct {
|
|
fs *Fs // what this object is part of
|
|
remote string // The remote path
|
|
id string // Drive Id of this object
|
|
url string // Download URL of this object
|
|
md5sum string // md5sum of the object
|
|
bytes int64 // size of the object
|
|
modifiedDate string // RFC3339 time it was last modified
|
|
isDocument bool // if set this is a Google doc
|
|
v2Download bool // generate v2 download link ondemand
|
|
mimeType string
|
|
}
|
|
|
|
// ------------------------------------------------------------
|
|
|
|
// Name of the remote (as passed into NewFs)
|
|
func (f *Fs) Name() string {
|
|
return f.name
|
|
}
|
|
|
|
// Root of the remote (as passed into NewFs)
|
|
func (f *Fs) Root() string {
|
|
return f.root
|
|
}
|
|
|
|
// String converts this Fs to a string
|
|
func (f *Fs) String() string {
|
|
return fmt.Sprintf("Google drive root '%s'", f.root)
|
|
}
|
|
|
|
// Features returns the optional features of this Fs
|
|
func (f *Fs) Features() *fs.Features {
|
|
return f.features
|
|
}
|
|
|
|
// shouldRetry determines whehter a given err rates being retried
|
|
func shouldRetry(err error) (again bool, errOut error) {
|
|
again = false
|
|
if err != nil {
|
|
if fserrors.ShouldRetry(err) {
|
|
again = true
|
|
} else {
|
|
switch gerr := err.(type) {
|
|
case *googleapi.Error:
|
|
if gerr.Code >= 500 && gerr.Code < 600 {
|
|
// All 5xx errors should be retried
|
|
again = true
|
|
} else if len(gerr.Errors) > 0 {
|
|
reason := gerr.Errors[0].Reason
|
|
if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {
|
|
again = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return again, err
|
|
}
|
|
|
|
// parseParse parses a drive 'url'
|
|
func parseDrivePath(path string) (root string, err error) {
|
|
root = strings.Trim(path, "/")
|
|
return
|
|
}
|
|
|
|
// User function to process a File item from list
|
|
//
|
|
// Should return true to finish processing
|
|
type listFn func(*drive.File) bool
|
|
|
|
func containsString(slice []string, s string) bool {
|
|
for _, e := range slice {
|
|
if e == s {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Lists the directory required calling the user function on each item found
|
|
//
|
|
// If the user fn ever returns true then it early exits with found = true
|
|
//
|
|
// Search params: https://developers.google.com/drive/search-parameters
|
|
func (f *Fs) list(dirIDs []string, title string, directoriesOnly bool, filesOnly bool, includeAll bool, fn listFn) (found bool, err error) {
|
|
var query []string
|
|
if !includeAll {
|
|
q := "trashed=" + strconv.FormatBool(f.opt.TrashedOnly)
|
|
if f.opt.TrashedOnly {
|
|
q = fmt.Sprintf("(mimeType='%s' or %s)", driveFolderType, q)
|
|
}
|
|
query = append(query, q)
|
|
}
|
|
// Search with sharedWithMe will always return things listed in "Shared With Me" (without any parents)
|
|
// We must not filter with parent when we try list "ROOT" with drive-shared-with-me
|
|
// If we need to list file inside those shared folders, we must search it without sharedWithMe
|
|
parentsQuery := bytes.NewBufferString("(")
|
|
for _, dirID := range dirIDs {
|
|
if dirID == "" {
|
|
continue
|
|
}
|
|
if parentsQuery.Len() > 1 {
|
|
_, _ = parentsQuery.WriteString(" or ")
|
|
}
|
|
if f.opt.SharedWithMe && dirID == f.rootFolderID {
|
|
_, _ = parentsQuery.WriteString("sharedWithMe=true")
|
|
} else {
|
|
_, _ = fmt.Fprintf(parentsQuery, "'%s' in parents", dirID)
|
|
}
|
|
}
|
|
if parentsQuery.Len() > 1 {
|
|
_ = parentsQuery.WriteByte(')')
|
|
query = append(query, parentsQuery.String())
|
|
}
|
|
stem := ""
|
|
if title != "" {
|
|
// Escaping the backslash isn't documented but seems to work
|
|
searchTitle := strings.Replace(title, `\`, `\\`, -1)
|
|
searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
|
|
// Convert ／ to / for search
|
|
searchTitle = strings.Replace(searchTitle, "／", "/", -1)
|
|
|
|
handleGdocs := !directoriesOnly && !f.opt.SkipGdocs
|
|
// if the search title contains an extension and the extension is in the export extensions add a search
|
|
// for the filename without the extension.
|
|
// assume that export extensions don't contain escape sequences and only have one part (not .tar.gz)
|
|
if ext := path.Ext(searchTitle); handleGdocs && len(ext) > 0 && containsString(f.extensions, ext) {
|
|
stem = title[:len(title)-len(ext)]
|
|
query = append(query, fmt.Sprintf("(name='%s' or name='%s')", searchTitle, searchTitle[:len(searchTitle)-len(ext)]))
|
|
} else {
|
|
query = append(query, fmt.Sprintf("name='%s'", searchTitle))
|
|
}
|
|
}
|
|
if directoriesOnly {
|
|
query = append(query, fmt.Sprintf("mimeType='%s'", driveFolderType))
|
|
}
|
|
if filesOnly {
|
|
query = append(query, fmt.Sprintf("mimeType!='%s'", driveFolderType))
|
|
}
|
|
list := f.svc.Files.List()
|
|
if len(query) > 0 {
|
|
list.Q(strings.Join(query, " and "))
|
|
// fmt.Printf("list Query = %q\n", query)
|
|
}
|
|
if f.opt.ListChunk > 0 {
|
|
list.PageSize(f.opt.ListChunk)
|
|
}
|
|
if f.isTeamDrive {
|
|
list.TeamDriveId(f.opt.TeamDriveID)
|
|
list.SupportsTeamDrives(true)
|
|
list.IncludeTeamDriveItems(true)
|
|
list.Corpora("teamDrive")
|
|
}
|
|
// If using appDataFolder then need to add Spaces
|
|
if f.rootFolderID == "appDataFolder" {
|
|
list.Spaces("appDataFolder")
|
|
}
|
|
|
|
var fields = partialFields
|
|
|
|
if f.opt.AuthOwnerOnly {
|
|
fields += ",owners"
|
|
}
|
|
|
|
fields = fmt.Sprintf("files(%s),nextPageToken", fields)
|
|
|
|
OUTER:
|
|
for {
|
|
var files *drive.FileList
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
files, err = list.Fields(googleapi.Field(fields)).Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return false, errors.Wrap(err, "couldn't list directory")
|
|
}
|
|
for _, item := range files.Files {
|
|
// Convert / to ／ for listing purposes
|
|
item.Name = strings.Replace(item.Name, "/", "／", -1)
|
|
// Check the case of items is correct since
|
|
// the `=` operator is case insensitive.
|
|
|
|
if title != "" && title != item.Name {
|
|
if stem == "" || stem != item.Name {
|
|
continue
|
|
}
|
|
_, exportName, _, _ := f.findExportFormat(item)
|
|
if exportName == "" || exportName != title {
|
|
continue
|
|
}
|
|
}
|
|
if fn(item) {
|
|
found = true
|
|
break OUTER
|
|
}
|
|
}
|
|
if files.NextPageToken == "" {
|
|
break
|
|
}
|
|
list.PageToken(files.NextPageToken)
|
|
}
|
|
return
|
|
}
|
|
|
|
// Returns true of x is a power of 2 or zero
|
|
func isPowerOfTwo(x int64) bool {
|
|
switch {
|
|
case x == 0:
|
|
return true
|
|
case x < 0:
|
|
return false
|
|
default:
|
|
return (x & (x - 1)) == 0
|
|
}
|
|
}
|
|
|
|
// add a charset parameter to all text/* MIME types
|
|
func fixMimeType(mimeType string) string {
|
|
mediaType, param, err := mime.ParseMediaType(mimeType)
|
|
if err != nil {
|
|
return mimeType
|
|
}
|
|
if strings.HasPrefix(mimeType, "text/") && param["charset"] == "" {
|
|
param["charset"] = "utf-8"
|
|
mimeType = mime.FormatMediaType(mediaType, param)
|
|
}
|
|
return mimeType
|
|
}
|
|
func fixMimeTypeMap(m map[string][]string) map[string][]string {
|
|
for _, v := range m {
|
|
for i, mt := range v {
|
|
fixed := fixMimeType(mt)
|
|
if fixed == "" {
|
|
panic(errors.Errorf("unable to fix MIME type %q", mt))
|
|
}
|
|
v[i] = fixed
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
func isInternalMimeType(mimeType string) bool {
|
|
return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
|
|
}
|
|
|
|
// parseExtensions parses a list of comma separated extensions
|
|
// into a list of unique extensions with leading "."
|
|
func parseExtensions(extensions ...string) ([]string, error) {
|
|
var result []string
|
|
for _, extensionText := range extensions {
|
|
for _, extension := range strings.Split(extensionText, ",") {
|
|
extension = strings.ToLower(strings.TrimSpace(extension))
|
|
if len(extension) > 0 && extension[0] != '.' {
|
|
extension = "." + extension
|
|
}
|
|
if mime.TypeByExtension(extension) == "" {
|
|
return result, errors.Errorf("couldn't find MIME type for extension %q", extension)
|
|
}
|
|
found := false
|
|
for _, existingExtension := range result {
|
|
if extension == existingExtension {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
result = append(result, extension)
|
|
}
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// parseExtensionMimeTypes parses the given extensions using parseExtensions
|
|
// and maps each resulting extension to its MIME type.
|
|
func parseExtensionMimeTypes(extensions ...string) ([]string, error) {
|
|
parsedExtensions, err := parseExtensions(extensions...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
mimeTypes := make([]string, 0, len(parsedExtensions))
|
|
for i, extension := range parsedExtensions {
|
|
mt := mime.TypeByExtension(extension)
|
|
if mt == "" {
|
|
return nil, errors.Errorf("couldn't find MIME type for extension %q", extension)
|
|
}
|
|
mimeTypes[i] = mt
|
|
}
|
|
return mimeTypes, nil
|
|
}
|
|
|
|
// Figure out if the user wants to use a team drive
|
|
func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
|
|
if opt.TeamDriveID == "" {
|
|
fmt.Printf("Configure this as a team drive?\n")
|
|
} else {
|
|
fmt.Printf("Change current team drive ID %q?\n", opt.TeamDriveID)
|
|
}
|
|
if !config.ConfirmWithDefault(false) {
|
|
return nil
|
|
}
|
|
client, err := createOAuthClient(opt, name, m)
|
|
if err != nil {
|
|
return errors.Wrap(err, "config team drive failed to create oauth client")
|
|
}
|
|
svc, err := drive.New(client)
|
|
if err != nil {
|
|
return errors.Wrap(err, "config team drive failed to make drive client")
|
|
}
|
|
fmt.Printf("Fetching team drive list...\n")
|
|
var driveIDs, driveNames []string
|
|
listTeamDrives := svc.Teamdrives.List().PageSize(100)
|
|
for {
|
|
var teamDrives *drive.TeamDriveList
|
|
err = newPacer().Call(func() (bool, error) {
|
|
teamDrives, err = listTeamDrives.Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return errors.Wrap(err, "list team drives failed")
|
|
}
|
|
for _, drive := range teamDrives.TeamDrives {
|
|
driveIDs = append(driveIDs, drive.Id)
|
|
driveNames = append(driveNames, drive.Name)
|
|
}
|
|
if teamDrives.NextPageToken == "" {
|
|
break
|
|
}
|
|
listTeamDrives.PageToken(teamDrives.NextPageToken)
|
|
}
|
|
var driveID string
|
|
if len(driveIDs) == 0 {
|
|
fmt.Printf("No team drives found in your account")
|
|
} else {
|
|
driveID = config.Choose("Enter a Team Drive ID", driveIDs, driveNames, true)
|
|
}
|
|
m.Set("team_drive", driveID)
|
|
opt.TeamDriveID = driveID
|
|
return nil
|
|
}
|
|
|
|
// newPacer makes a pacer configured for drive
|
|
func newPacer() *pacer.Pacer {
|
|
return pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer)
|
|
}
|
|
|
|
func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {
|
|
conf, err := google.JWTConfigFromJSON(credentialsData, driveConfig.Scopes...)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "error processing credentials")
|
|
}
|
|
if opt.Impersonate != "" {
|
|
conf.Subject = opt.Impersonate
|
|
}
|
|
ctxWithSpecialClient := oauthutil.Context(fshttp.NewClient(fs.Config))
|
|
return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
|
|
}
|
|
|
|
func createOAuthClient(opt *Options, name string, m configmap.Mapper) (*http.Client, error) {
|
|
var oAuthClient *http.Client
|
|
var err error
|
|
|
|
// try loading service account credentials from env variable, then from a file
|
|
if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
|
|
loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile))
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "error opening service account credentials file")
|
|
}
|
|
opt.ServiceAccountCredentials = string(loadedCreds)
|
|
}
|
|
if opt.ServiceAccountCredentials != "" {
|
|
oAuthClient, err = getServiceAccountClient(opt, []byte(opt.ServiceAccountCredentials))
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to create oauth client from service account")
|
|
}
|
|
} else {
|
|
oAuthClient, _, err = oauthutil.NewClient(name, m, driveConfig)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to create oauth client")
|
|
}
|
|
}
|
|
|
|
return oAuthClient, nil
|
|
}
|
|
|
|
// NewFs contstructs an Fs from the path, container:path
|
|
func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
|
|
// Parse config into Options struct
|
|
opt := new(Options)
|
|
err := configstruct.Set(m, opt)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !isPowerOfTwo(int64(opt.ChunkSize)) {
|
|
return nil, errors.Errorf("drive: chunk size %v isn't a power of two", opt.ChunkSize)
|
|
}
|
|
if opt.ChunkSize < 256*1024 {
|
|
return nil, errors.Errorf("drive: chunk size can't be less than 256k - was %v", opt.ChunkSize)
|
|
}
|
|
|
|
oAuthClient, err := createOAuthClient(opt, name, m)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "drive: failed when making oauth client")
|
|
}
|
|
|
|
root, err := parseDrivePath(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
f := &Fs{
|
|
name: name,
|
|
root: root,
|
|
opt: *opt,
|
|
pacer: newPacer(),
|
|
}
|
|
f.isTeamDrive = opt.TeamDriveID != ""
|
|
f.features = (&fs.Features{
|
|
DuplicateFiles: true,
|
|
ReadMimeType: true,
|
|
WriteMimeType: true,
|
|
CanHaveEmptyDirectories: true,
|
|
}).Fill(f)
|
|
|
|
// Create a new authorized Drive client.
|
|
f.client = oAuthClient
|
|
f.svc, err = drive.New(f.client)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "couldn't create Drive client")
|
|
}
|
|
|
|
if f.opt.V2DownloadMinSize >= 0 {
|
|
f.v2Svc, err = drive_v2.New(f.client)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "couldn't create Drive v2 client")
|
|
}
|
|
}
|
|
|
|
// set root folder for a team drive or query the user root folder
|
|
if f.isTeamDrive {
|
|
f.rootFolderID = f.opt.TeamDriveID
|
|
} else {
|
|
f.rootFolderID = "root"
|
|
}
|
|
|
|
// override root folder if set in the config
|
|
if opt.RootFolderID != "" {
|
|
f.rootFolderID = opt.RootFolderID
|
|
}
|
|
|
|
f.dirCache = dircache.New(root, f.rootFolderID, f)
|
|
|
|
// Parse extensions
|
|
f.extensions, err = parseExtensions(opt.Extensions, defaultExtensions)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Find the current root
|
|
err = f.dirCache.FindRoot(false)
|
|
if err != nil {
|
|
// Assume it is a file
|
|
newRoot, remote := dircache.SplitPath(root)
|
|
tempF := *f
|
|
tempF.dirCache = dircache.New(newRoot, f.rootFolderID, &tempF)
|
|
tempF.root = newRoot
|
|
// Make new Fs which is the parent
|
|
err = tempF.dirCache.FindRoot(false)
|
|
if err != nil {
|
|
// No root so return old f
|
|
return f, nil
|
|
}
|
|
_, err := tempF.NewObject(remote)
|
|
if err != nil {
|
|
// unable to list folder so return old f
|
|
return f, nil
|
|
}
|
|
// XXX: update the old f here instead of returning tempF, since
|
|
// `features` were already filled with functions having *f as a receiver.
|
|
// See https://github.com/ncw/rclone/issues/2182
|
|
f.dirCache = tempF.dirCache
|
|
f.root = tempF.root
|
|
return f, fs.ErrorIsFile
|
|
}
|
|
// fmt.Printf("Root id %s", f.dirCache.RootID())
|
|
return f, nil
|
|
}
|
|
|
|
// Return an Object from a path
|
|
//
|
|
// If it can't be found it returns the error fs.ErrorObjectNotFound.
|
|
func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
|
|
o := &Object{
|
|
fs: f,
|
|
remote: remote,
|
|
}
|
|
if info != nil {
|
|
o.setMetaData(info)
|
|
} else {
|
|
err := o.readMetaData() // reads info and meta, returning an error
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return o, nil
|
|
}
|
|
|
|
// NewObject finds the Object at remote. If it can't be found
|
|
// it returns the error fs.ErrorObjectNotFound.
|
|
func (f *Fs) NewObject(remote string) (fs.Object, error) {
|
|
return f.newObjectWithInfo(remote, nil)
|
|
}
|
|
|
|
// FindLeaf finds a directory of name leaf in the folder with ID pathID
|
|
func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) {
|
|
// Find the leaf in pathID
|
|
found, err = f.list([]string{pathID}, leaf, true, false, false, func(item *drive.File) bool {
|
|
if item.Name == leaf {
|
|
pathIDOut = item.Id
|
|
return true
|
|
}
|
|
if !f.opt.SkipGdocs {
|
|
_, exportName, _, _ := f.findExportFormat(item)
|
|
if exportName == leaf {
|
|
pathIDOut = item.Id
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
})
|
|
return pathIDOut, found, err
|
|
}
|
|
|
|
// CreateDir makes a directory with pathID as parent and name leaf
|
|
func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) {
|
|
// fmt.Println("Making", path)
|
|
// Define the metadata for the directory we are going to create.
|
|
createInfo := &drive.File{
|
|
Name: leaf,
|
|
Description: leaf,
|
|
MimeType: driveFolderType,
|
|
Parents: []string{pathID},
|
|
}
|
|
var info *drive.File
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
info, err = f.svc.Files.Create(createInfo).
|
|
Fields(googleapi.Field(partialFields)).
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return info.Id, nil
|
|
}
|
|
|
|
// isAuthOwned checks if any of the item owners is the authenticated owner
|
|
func isAuthOwned(item *drive.File) bool {
|
|
for _, owner := range item.Owners {
|
|
if owner.Me {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// exportFormats returns the export formats from drive, fetching them
|
|
// if necessary.
|
|
//
|
|
// if the fetch fails then it will not export any drive formats
|
|
func (f *Fs) exportFormats() map[string][]string {
|
|
exportFormatsOnce.Do(func() {
|
|
var about *drive.About
|
|
var err error
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
about, err = f.svc.About.Get().
|
|
Fields("exportFormats").
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
fs.Errorf(f, "Failed to get Drive exportFormats: %v", err)
|
|
_exportFormats = map[string][]string{}
|
|
return
|
|
}
|
|
_exportFormats = fixMimeTypeMap(about.ExportFormats)
|
|
})
|
|
return _exportFormats
|
|
}
|
|
|
|
// findExportFormat works out the optimum extension and MIME type
|
|
// for this item.
|
|
//
|
|
// Look through the extensions and find the first format that can be
|
|
// converted. If none found then return "", ""
|
|
func (f *Fs) findExportFormat(item *drive.File) (extension, filename, mimeType string, isDocument bool) {
|
|
exportMimeTypes, isDocument := f.exportFormats()[item.MimeType]
|
|
if isDocument {
|
|
for _, _extension := range f.extensions {
|
|
_mimeType := mime.TypeByExtension(_extension)
|
|
for _, emt := range exportMimeTypes {
|
|
if emt == _mimeType {
|
|
return _extension, item.Name + _extension, _mimeType, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// else return empty
|
|
return "", "", "", isDocument
|
|
}
|
|
|
|
// List the objects and directories in dir into entries. The
|
|
// entries can be returned in any order but should be for a
|
|
// complete directory.
|
|
//
|
|
// dir should be "" to list the root, and should not have
|
|
// trailing slashes.
|
|
//
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
// found.
|
|
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
|
err = f.dirCache.FindRoot(false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
directoryID, err := f.dirCache.FindDir(dir, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var iErr error
|
|
_, err = f.list([]string{directoryID}, "", false, false, false, func(item *drive.File) bool {
|
|
entry, err := f.itemToDirEntry(path.Join(dir, item.Name), item)
|
|
if err != nil {
|
|
return true
|
|
}
|
|
if entry != nil {
|
|
entries = append(entries, entry)
|
|
}
|
|
return false
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if iErr != nil {
|
|
return nil, iErr
|
|
}
|
|
return entries, nil
|
|
}
|
|
|
|
// listRRunner will read dirIDs from the in channel, perform the file listing an call cb with each DirEntry.
|
|
//
|
|
// In each cycle, will wait up to 10ms to read up to grouping entries from the in channel.
|
|
// If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
|
|
// nil is send to the out channel and the function returns.
|
|
func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan string, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
|
|
var dirs []string
|
|
|
|
for dir := range in {
|
|
dirs = append(dirs[:0], dir)
|
|
wait := time.After(10 * time.Millisecond)
|
|
waitloop:
|
|
for i := 1; i < grouping; i++ {
|
|
select {
|
|
case d, ok := <-in:
|
|
if !ok {
|
|
break waitloop
|
|
}
|
|
dirs = append(dirs, d)
|
|
case <-wait:
|
|
break waitloop
|
|
}
|
|
}
|
|
var iErr error
|
|
_, err := f.list(dirs, "", false, false, false, func(item *drive.File) bool {
|
|
parentPath := ""
|
|
if len(item.Parents) > 0 {
|
|
p, ok := f.dirCache.GetInv(item.Parents[0])
|
|
if ok {
|
|
parentPath = p
|
|
}
|
|
}
|
|
remote := path.Join(parentPath, item.Name)
|
|
entry, err := f.itemToDirEntry(remote, item)
|
|
if err != nil {
|
|
iErr = err
|
|
return true
|
|
}
|
|
|
|
err = cb(entry)
|
|
if err != nil {
|
|
iErr = err
|
|
return true
|
|
}
|
|
return false
|
|
})
|
|
for range dirs {
|
|
wg.Done()
|
|
}
|
|
|
|
if iErr != nil {
|
|
out <- iErr
|
|
return
|
|
}
|
|
|
|
if err != nil {
|
|
out <- err
|
|
return
|
|
}
|
|
}
|
|
out <- nil
|
|
}
|
|
|
|
// ListR lists the objects and directories of the Fs starting
|
|
// from dir recursively into out.
|
|
//
|
|
// dir should be "" to start from the root, and should not
|
|
// have trailing slashes.
|
|
//
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
// found.
|
|
//
|
|
// It should call callback for each tranche of entries read.
|
|
// These need not be returned in any particular order. If
|
|
// callback returns an error then the listing will stop
|
|
// immediately.
|
|
//
|
|
// Don't implement this unless you have a more efficient way
|
|
// of listing recursively that doing a directory traversal.
|
|
func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
|
|
const (
|
|
grouping = 50
|
|
inputBuffer = 1000
|
|
)
|
|
|
|
err = f.dirCache.FindRoot(false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
directoryID, err := f.dirCache.FindDir(dir, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
mu := sync.Mutex{} // protects in and overflow
|
|
wg := sync.WaitGroup{}
|
|
in := make(chan string, inputBuffer)
|
|
out := make(chan error, fs.Config.Checkers)
|
|
list := walk.NewListRHelper(callback)
|
|
overfflow := []string{}
|
|
|
|
cb := func(entry fs.DirEntry) error {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if d, isDir := entry.(*fs.Dir); isDir && in != nil {
|
|
select {
|
|
case in <- d.ID():
|
|
wg.Add(1)
|
|
default:
|
|
overfflow = append(overfflow, d.ID())
|
|
}
|
|
}
|
|
return list.Add(entry)
|
|
}
|
|
|
|
wg.Add(1)
|
|
in <- directoryID
|
|
|
|
for i := 0; i < fs.Config.Checkers; i++ {
|
|
go f.listRRunner(&wg, in, out, cb, grouping)
|
|
}
|
|
go func() {
|
|
// wait until the all directories are processed
|
|
wg.Wait()
|
|
// if the input channel overflowed add the collected entries to the channel now
|
|
for len(overfflow) > 0 {
|
|
mu.Lock()
|
|
l := len(overfflow)
|
|
// only fill half of the channel to prevent entries beeing put into overfflow again
|
|
if l > inputBuffer/2 {
|
|
l = inputBuffer / 2
|
|
}
|
|
wg.Add(l)
|
|
for _, d := range overfflow[:l] {
|
|
in <- d
|
|
}
|
|
overfflow = overfflow[l:]
|
|
mu.Unlock()
|
|
|
|
// wait again for the completion of all directories
|
|
wg.Wait()
|
|
}
|
|
mu.Lock()
|
|
if in != nil {
|
|
// notify all workers to exit
|
|
close(in)
|
|
in = nil
|
|
}
|
|
mu.Unlock()
|
|
}()
|
|
// wait until the all workers to finish
|
|
for i := 0; i < fs.Config.Checkers; i++ {
|
|
e := <-out
|
|
mu.Lock()
|
|
// if one worker returns an error early, close the input so all other workers exit
|
|
if e != nil && in != nil {
|
|
err = e
|
|
close(in)
|
|
in = nil
|
|
}
|
|
mu.Unlock()
|
|
}
|
|
|
|
close(out)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return list.Flush()
|
|
}
|
|
|
|
func (f *Fs) itemToDirEntry(remote string, item *drive.File) (fs.DirEntry, error) {
|
|
switch {
|
|
case item.MimeType == driveFolderType:
|
|
// cache the directory ID for later lookups
|
|
f.dirCache.Put(remote, item.Id)
|
|
when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
|
|
d := fs.NewDir(remote, when).SetID(item.Id)
|
|
return d, nil
|
|
case f.opt.AuthOwnerOnly && !isAuthOwned(item):
|
|
// ignore object
|
|
case item.Md5Checksum != "" || item.Size > 0:
|
|
// If item has MD5 sum or a length it is a file stored on drive
|
|
o, err := f.newObjectWithInfo(remote, item)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return o, nil
|
|
case f.opt.SkipGdocs:
|
|
fs.Debugf(remote, "Skipping google document type %q", item.MimeType)
|
|
default:
|
|
// If item MimeType is in the ExportFormats then it is a google doc
|
|
extension, _, exportMimeType, isDocument := f.findExportFormat(item)
|
|
if !isDocument {
|
|
fs.Debugf(remote, "Ignoring unknown document type %q", item.MimeType)
|
|
break
|
|
}
|
|
if extension == "" {
|
|
fs.Debugf(remote, "No export formats found for %q", item.MimeType)
|
|
break
|
|
}
|
|
o, err := f.newObjectWithInfo(remote+extension, item)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
o.(*Object).setGdocsMetaData(item, extension, exportMimeType)
|
|
return o, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// Creates a drive.File info from the parameters passed in and a half
|
|
// finished Object which must have setMetaData called on it
|
|
//
|
|
// Used to create new objects
|
|
func (f *Fs) createFileInfo(remote string, modTime time.Time, size int64) (*Object, *drive.File, error) {
|
|
// Temporary Object under construction
|
|
o := &Object{
|
|
fs: f,
|
|
remote: remote,
|
|
bytes: size,
|
|
}
|
|
|
|
leaf, directoryID, err := f.dirCache.FindRootAndPath(remote, true)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
// Define the metadata for the file we are going to create.
|
|
createInfo := &drive.File{
|
|
Name: leaf,
|
|
Description: leaf,
|
|
Parents: []string{directoryID},
|
|
MimeType: fs.MimeTypeFromName(remote),
|
|
ModifiedTime: modTime.Format(timeFormatOut),
|
|
}
|
|
return o, createInfo, nil
|
|
}
|
|
|
|
// Put the object
|
|
//
|
|
// Copy the reader in to the new object which is returned
|
|
//
|
|
// The new object may have been created if an error is returned
|
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
|
|
switch err {
|
|
case nil:
|
|
return exisitingObj, exisitingObj.Update(in, src, options...)
|
|
case fs.ErrorObjectNotFound:
|
|
// Not found so create it
|
|
return f.PutUnchecked(in, src, options...)
|
|
default:
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
|
func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
return f.Put(in, src, options...)
|
|
}
|
|
|
|
// PutUnchecked uploads the object
|
|
//
|
|
// This will create a duplicate if we upload a new file without
|
|
// checking to see if there is one already - use Put() for that.
|
|
func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
remote := src.Remote()
|
|
size := src.Size()
|
|
modTime := src.ModTime()
|
|
|
|
o, createInfo, err := f.createFileInfo(remote, modTime, size)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var info *drive.File
|
|
if size == 0 || size < int64(f.opt.UploadCutoff) {
|
|
// Make the API request to upload metadata and file data.
|
|
// Don't retry, return a retry error instead
|
|
err = f.pacer.CallNoRetry(func() (bool, error) {
|
|
info, err = f.svc.Files.Create(createInfo).
|
|
Media(in, googleapi.ContentType("")).
|
|
Fields(googleapi.Field(partialFields)).
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
KeepRevisionForever(f.opt.KeepRevisionForever).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return o, err
|
|
}
|
|
} else {
|
|
// Upload the file in chunks
|
|
info, err = f.Upload(in, size, createInfo.MimeType, "", createInfo, remote)
|
|
if err != nil {
|
|
return o, err
|
|
}
|
|
}
|
|
o.setMetaData(info)
|
|
return o, nil
|
|
}
|
|
|
|
// MergeDirs merges the contents of all the directories passed
|
|
// in into the first one and rmdirs the other directories.
|
|
func (f *Fs) MergeDirs(dirs []fs.Directory) error {
|
|
if len(dirs) < 2 {
|
|
return nil
|
|
}
|
|
dstDir := dirs[0]
|
|
for _, srcDir := range dirs[1:] {
|
|
// list the the objects
|
|
infos := []*drive.File{}
|
|
_, err := f.list([]string{srcDir.ID()}, "", false, false, true, func(info *drive.File) bool {
|
|
infos = append(infos, info)
|
|
return false
|
|
})
|
|
if err != nil {
|
|
return errors.Wrapf(err, "MergeDirs list failed on %v", srcDir)
|
|
}
|
|
// move them into place
|
|
for _, info := range infos {
|
|
fs.Infof(srcDir, "merging %q", info.Name)
|
|
// Move the file into the destination
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
_, err = f.svc.Files.Update(info.Id, nil).
|
|
RemoveParents(srcDir.ID()).
|
|
AddParents(dstDir.ID()).
|
|
Fields("").
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.Name, srcDir)
|
|
}
|
|
}
|
|
// rmdir (into trash) the now empty source directory
|
|
fs.Infof(srcDir, "removing empty directory")
|
|
err = f.rmdir(srcDir.ID(), true)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Mkdir creates the container if it doesn't exist
|
|
func (f *Fs) Mkdir(dir string) error {
|
|
err := f.dirCache.FindRoot(true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if dir != "" {
|
|
_, err = f.dirCache.FindDir(dir, true)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Rmdir deletes a directory unconditionally by ID
|
|
func (f *Fs) rmdir(directoryID string, useTrash bool) error {
|
|
return f.pacer.Call(func() (bool, error) {
|
|
var err error
|
|
if useTrash {
|
|
info := drive.File{
|
|
Trashed: true,
|
|
}
|
|
_, err = f.svc.Files.Update(directoryID, &info).
|
|
Fields("").
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
} else {
|
|
err = f.svc.Files.Delete(directoryID).
|
|
Fields("").
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
}
|
|
return shouldRetry(err)
|
|
})
|
|
}
|
|
|
|
// Rmdir deletes a directory
|
|
//
|
|
// Returns an error if it isn't empty
|
|
func (f *Fs) Rmdir(dir string) error {
|
|
root := path.Join(f.root, dir)
|
|
dc := f.dirCache
|
|
directoryID, err := dc.FindDir(dir, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var trashedFiles = false
|
|
found, err := f.list([]string{directoryID}, "", false, false, true, func(item *drive.File) bool {
|
|
if !item.Trashed {
|
|
fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
|
|
return true
|
|
}
|
|
fs.Debugf(dir, "Rmdir: contains trashed file: %q", item.Name)
|
|
trashedFiles = true
|
|
return false
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if found {
|
|
return errors.Errorf("directory not empty")
|
|
}
|
|
if root != "" {
|
|
// trash the directory if it had trashed files
|
|
// in or the user wants to trash, otherwise
|
|
// delete it.
|
|
err = f.rmdir(directoryID, trashedFiles || f.opt.UseTrash)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
f.dirCache.FlushDir(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Precision of the object storage system
|
|
func (f *Fs) Precision() time.Duration {
|
|
return time.Millisecond
|
|
}
|
|
|
|
// Copy src to this remote using server side copy operations.
|
|
//
|
|
// This is stored with the remote path given
|
|
//
|
|
// It returns the destination Object and a possible error
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
|
func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
|
|
srcObj, ok := src.(*Object)
|
|
if !ok {
|
|
fs.Debugf(src, "Can't copy - not same remote type")
|
|
return nil, fs.ErrorCantCopy
|
|
}
|
|
if srcObj.isDocument {
|
|
return nil, errors.New("can't copy a Google document")
|
|
}
|
|
|
|
o, createInfo, err := f.createFileInfo(remote, srcObj.ModTime(), srcObj.bytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var info *drive.File
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
|
info, err = o.fs.svc.Files.Copy(srcObj.id, createInfo).
|
|
Fields(googleapi.Field(partialFields)).
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
KeepRevisionForever(f.opt.KeepRevisionForever).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
o.setMetaData(info)
|
|
return o, nil
|
|
}
|
|
|
|
// Purge deletes all the files and the container
|
|
//
|
|
// Optional interface: Only implement this if you have a way of
|
|
// deleting all the files quicker than just running Remove() on the
|
|
// result of List()
|
|
func (f *Fs) Purge() error {
|
|
if f.root == "" {
|
|
return errors.New("can't purge root directory")
|
|
}
|
|
err := f.dirCache.FindRoot(false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
if f.opt.UseTrash {
|
|
info := drive.File{
|
|
Trashed: true,
|
|
}
|
|
_, err = f.svc.Files.Update(f.dirCache.RootID(), &info).
|
|
Fields("").
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
} else {
|
|
err = f.svc.Files.Delete(f.dirCache.RootID()).
|
|
Fields("").
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
}
|
|
return shouldRetry(err)
|
|
})
|
|
f.dirCache.ResetRoot()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CleanUp empties the trash
|
|
func (f *Fs) CleanUp() error {
|
|
err := f.pacer.Call(func() (bool, error) {
|
|
err := f.svc.Files.EmptyTrash().Do()
|
|
return shouldRetry(err)
|
|
})
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// About gets quota information
|
|
func (f *Fs) About() (*fs.Usage, error) {
|
|
if f.isTeamDrive {
|
|
// Teamdrives don't appear to have a usage API so just return empty
|
|
return &fs.Usage{}, nil
|
|
}
|
|
var about *drive.About
|
|
var err error
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
about, err = f.svc.About.Get().Fields("storageQuota").Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to get Drive storageQuota")
|
|
}
|
|
q := about.StorageQuota
|
|
usage := &fs.Usage{
|
|
Used: fs.NewUsageValue(q.UsageInDrive), // bytes in use
|
|
Trashed: fs.NewUsageValue(q.UsageInDriveTrash), // bytes in trash
|
|
Other: fs.NewUsageValue(q.Usage - q.UsageInDrive), // other usage eg gmail in drive
|
|
}
|
|
if q.Limit > 0 {
|
|
usage.Total = fs.NewUsageValue(q.Limit) // quota of bytes that can be used
|
|
usage.Free = fs.NewUsageValue(q.Limit - q.Usage) // bytes which can be uploaded before reaching the quota
|
|
}
|
|
return usage, nil
|
|
}
|
|
|
|
// Move src to this remote using server side move operations.
|
|
//
|
|
// This is stored with the remote path given
|
|
//
|
|
// It returns the destination Object and a possible error
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantMove
|
|
func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
|
|
srcObj, ok := src.(*Object)
|
|
if !ok {
|
|
fs.Debugf(src, "Can't move - not same remote type")
|
|
return nil, fs.ErrorCantMove
|
|
}
|
|
if srcObj.isDocument {
|
|
return nil, errors.New("can't move a Google document")
|
|
}
|
|
_, srcParentID, err := srcObj.fs.dirCache.FindPath(src.Remote(), false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Temporary Object under construction
|
|
dstObj, dstInfo, err := f.createFileInfo(remote, srcObj.ModTime(), srcObj.bytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
dstParents := strings.Join(dstInfo.Parents, ",")
|
|
dstInfo.Parents = nil
|
|
|
|
// Do the move
|
|
var info *drive.File
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
info, err = f.svc.Files.Update(srcObj.id, dstInfo).
|
|
RemoveParents(srcParentID).
|
|
AddParents(dstParents).
|
|
Fields(googleapi.Field(partialFields)).
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
dstObj.setMetaData(info)
|
|
return dstObj, nil
|
|
}
|
|
|
|
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
|
|
func (f *Fs) PublicLink(remote string) (link string, err error) {
|
|
id, err := f.dirCache.FindDir(remote, false)
|
|
if err == nil {
|
|
fs.Debugf(f, "attempting to share directory '%s'", remote)
|
|
} else {
|
|
fs.Debugf(f, "attempting to share single file '%s'", remote)
|
|
o := &Object{
|
|
fs: f,
|
|
remote: remote,
|
|
}
|
|
if err = o.readMetaData(); err != nil {
|
|
return
|
|
}
|
|
id = o.id
|
|
}
|
|
|
|
permission := &drive.Permission{
|
|
AllowFileDiscovery: false,
|
|
Role: "reader",
|
|
Type: "anyone",
|
|
}
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
// TODO: On TeamDrives this might fail if lacking permissions to change ACLs.
|
|
// Need to either check `canShare` attribute on the object or see if a sufficient permission is already present.
|
|
_, err = f.svc.Permissions.Create(id, permission).
|
|
Fields(googleapi.Field("id")).
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return fmt.Sprintf("https://drive.google.com/open?id=%s", id), nil
|
|
}
|
|
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
|
// using server side move operations.
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
//
|
|
// If destination exists then return fs.ErrorDirExists
|
|
func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
|
|
srcFs, ok := src.(*Fs)
|
|
if !ok {
|
|
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
|
return fs.ErrorCantDirMove
|
|
}
|
|
srcPath := path.Join(srcFs.root, srcRemote)
|
|
dstPath := path.Join(f.root, dstRemote)
|
|
|
|
// Refuse to move to or from the root
|
|
if srcPath == "" || dstPath == "" {
|
|
fs.Debugf(src, "DirMove error: Can't move root")
|
|
return errors.New("can't move root directory")
|
|
}
|
|
|
|
// find the root src directory
|
|
err := srcFs.dirCache.FindRoot(false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// find the root dst directory
|
|
if dstRemote != "" {
|
|
err = f.dirCache.FindRoot(true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
if f.dirCache.FoundRoot() {
|
|
return fs.ErrorDirExists
|
|
}
|
|
}
|
|
|
|
// Find ID of dst parent, creating subdirs if necessary
|
|
var leaf, dstDirectoryID string
|
|
findPath := dstRemote
|
|
if dstRemote == "" {
|
|
findPath = f.root
|
|
}
|
|
leaf, dstDirectoryID, err = f.dirCache.FindPath(findPath, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Check destination does not exist
|
|
if dstRemote != "" {
|
|
_, err = f.dirCache.FindDir(dstRemote, false)
|
|
if err == fs.ErrorDirNotFound {
|
|
// OK
|
|
} else if err != nil {
|
|
return err
|
|
} else {
|
|
return fs.ErrorDirExists
|
|
}
|
|
}
|
|
|
|
// Find ID of src parent
|
|
var srcDirectoryID string
|
|
if srcRemote == "" {
|
|
srcDirectoryID, err = srcFs.dirCache.RootParentID()
|
|
} else {
|
|
_, srcDirectoryID, err = srcFs.dirCache.FindPath(srcRemote, false)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Find ID of src
|
|
srcID, err := srcFs.dirCache.FindDir(srcRemote, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Do the move
|
|
patch := drive.File{
|
|
Name: leaf,
|
|
}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
_, err = f.svc.Files.Update(srcID, &patch).
|
|
RemoveParents(srcDirectoryID).
|
|
AddParents(dstDirectoryID).
|
|
Fields("").
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
srcFs.dirCache.FlushDir(srcRemote)
|
|
return nil
|
|
}
|
|
|
|
// ChangeNotify calls the passed function with a path that has had changes.
|
|
// If the implementation uses polling, it should adhere to the given interval.
|
|
//
|
|
// Automatically restarts itself in case of unexpected behaviour of the remote.
|
|
//
|
|
// Close the returned channel to stop being notified.
|
|
func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
|
|
go func() {
|
|
// get the StartPageToken early so all changes from now on get processed
|
|
startPageToken, err := f.changeNotifyStartPageToken()
|
|
if err != nil {
|
|
fs.Infof(f, "Failed to get StartPageToken: %s", err)
|
|
}
|
|
var ticker *time.Ticker
|
|
var tickerC <-chan time.Time
|
|
for {
|
|
select {
|
|
case pollInterval, ok := <-pollIntervalChan:
|
|
if !ok {
|
|
if ticker != nil {
|
|
ticker.Stop()
|
|
}
|
|
return
|
|
}
|
|
if ticker != nil {
|
|
ticker.Stop()
|
|
ticker, tickerC = nil, nil
|
|
}
|
|
if pollInterval != 0 {
|
|
ticker = time.NewTicker(pollInterval)
|
|
tickerC = ticker.C
|
|
}
|
|
case <-tickerC:
|
|
if startPageToken == "" {
|
|
startPageToken, err = f.changeNotifyStartPageToken()
|
|
if err != nil {
|
|
fs.Infof(f, "Failed to get StartPageToken: %s", err)
|
|
continue
|
|
}
|
|
}
|
|
fs.Debugf(f, "Checking for changes on remote")
|
|
startPageToken, err = f.changeNotifyRunner(notifyFunc, startPageToken)
|
|
if err != nil {
|
|
fs.Infof(f, "Change notify listener failure: %s", err)
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
func (f *Fs) changeNotifyStartPageToken() (pageToken string, err error) {
|
|
var startPageToken *drive.StartPageToken
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
startPageToken, err = f.svc.Changes.GetStartPageToken().
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
return startPageToken.StartPageToken, nil
|
|
}
|
|
|
|
func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), startPageToken string) (newStartPageToken string, err error) {
|
|
pageToken := startPageToken
|
|
for {
|
|
var changeList *drive.ChangeList
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
changesCall := f.svc.Changes.List(pageToken).
|
|
Fields("nextPageToken,newStartPageToken,changes(fileId,file(name,parents,mimeType))")
|
|
if f.opt.ListChunk > 0 {
|
|
changesCall.PageSize(f.opt.ListChunk)
|
|
}
|
|
if f.isTeamDrive {
|
|
changesCall.TeamDriveId(f.opt.TeamDriveID)
|
|
changesCall.SupportsTeamDrives(true)
|
|
changesCall.IncludeTeamDriveItems(true)
|
|
}
|
|
changeList, err = changesCall.Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
type entryType struct {
|
|
path string
|
|
entryType fs.EntryType
|
|
}
|
|
var pathsToClear []entryType
|
|
for _, change := range changeList.Changes {
|
|
if path, ok := f.dirCache.GetInv(change.FileId); ok {
|
|
if change.File != nil && change.File.MimeType != driveFolderType {
|
|
pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryObject})
|
|
} else {
|
|
pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryDirectory})
|
|
}
|
|
continue
|
|
}
|
|
|
|
if change.File != nil {
|
|
changeType := fs.EntryDirectory
|
|
if change.File.MimeType != driveFolderType {
|
|
changeType = fs.EntryObject
|
|
}
|
|
|
|
// translate the parent dir of this object
|
|
if len(change.File.Parents) > 0 {
|
|
if path, ok := f.dirCache.GetInv(change.File.Parents[0]); ok {
|
|
// and append the drive file name to compute the full file name
|
|
if len(path) > 0 {
|
|
path = path + "/" + change.File.Name
|
|
} else {
|
|
path = change.File.Name
|
|
}
|
|
// this will now clear the actual file too
|
|
pathsToClear = append(pathsToClear, entryType{path: path, entryType: changeType})
|
|
}
|
|
} else { // a true root object that is changed
|
|
pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: changeType})
|
|
}
|
|
}
|
|
}
|
|
|
|
visitedPaths := make(map[string]bool)
|
|
for _, entry := range pathsToClear {
|
|
if _, ok := visitedPaths[entry.path]; ok {
|
|
continue
|
|
}
|
|
visitedPaths[entry.path] = true
|
|
notifyFunc(entry.path, entry.entryType)
|
|
}
|
|
|
|
switch {
|
|
case changeList.NewStartPageToken != "":
|
|
return changeList.NewStartPageToken, nil
|
|
case changeList.NextPageToken != "":
|
|
pageToken = changeList.NextPageToken
|
|
default:
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// DirCacheFlush resets the directory cache - used in testing as an
|
|
// optional interface
|
|
func (f *Fs) DirCacheFlush() {
|
|
f.dirCache.ResetRoot()
|
|
}
|
|
|
|
// Hashes returns the supported hash sets.
|
|
func (f *Fs) Hashes() hash.Set {
|
|
return hash.Set(hash.MD5)
|
|
}
|
|
|
|
// ------------------------------------------------------------
|
|
|
|
// Fs returns the parent Fs
|
|
func (o *Object) Fs() fs.Info {
|
|
return o.fs
|
|
}
|
|
|
|
// Return a string version
|
|
func (o *Object) String() string {
|
|
if o == nil {
|
|
return "<nil>"
|
|
}
|
|
return o.remote
|
|
}
|
|
|
|
// Remote returns the remote path
|
|
func (o *Object) Remote() string {
|
|
return o.remote
|
|
}
|
|
|
|
// Hash returns the Md5sum of an object returning a lowercase hex string
|
|
func (o *Object) Hash(t hash.Type) (string, error) {
|
|
if t != hash.MD5 {
|
|
return "", hash.ErrUnsupported
|
|
}
|
|
return o.md5sum, nil
|
|
}
|
|
|
|
// Size returns the size of an object in bytes
|
|
func (o *Object) Size() int64 {
|
|
return o.bytes
|
|
}
|
|
|
|
// setMetaData sets the fs data from a drive.File
|
|
func (o *Object) setMetaData(info *drive.File) {
|
|
o.id = info.Id
|
|
o.url = fmt.Sprintf("%sfiles/%s?alt=media", o.fs.svc.BasePath, info.Id)
|
|
o.md5sum = strings.ToLower(info.Md5Checksum)
|
|
o.bytes = info.Size
|
|
if o.bytes != -1 && o.fs.opt.V2DownloadMinSize != -1 && o.bytes >= int64(o.fs.opt.V2DownloadMinSize) {
|
|
o.v2Download = true
|
|
}
|
|
if o.fs.opt.UseCreatedDate {
|
|
o.modifiedDate = info.CreatedTime
|
|
} else {
|
|
o.modifiedDate = info.ModifiedTime
|
|
}
|
|
o.mimeType = info.MimeType
|
|
}
|
|
|
|
// setGdocsMetaData only sets the gdocs related fields
|
|
func (o *Object) setGdocsMetaData(info *drive.File, extension, exportMimeType string) {
|
|
o.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", o.fs.svc.BasePath, info.Id, url.QueryEscape(exportMimeType))
|
|
if o.fs.opt.AlternateExport {
|
|
switch info.MimeType {
|
|
case "application/vnd.google-apps.drawing":
|
|
o.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", info.Id, extension[1:])
|
|
case "application/vnd.google-apps.document":
|
|
o.url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", info.Id, extension[1:])
|
|
case "application/vnd.google-apps.spreadsheet":
|
|
o.url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", info.Id, extension[1:])
|
|
case "application/vnd.google-apps.presentation":
|
|
o.url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", info.Id, extension[1:])
|
|
}
|
|
}
|
|
o.isDocument = true
|
|
o.mimeType = exportMimeType
|
|
o.bytes = -1
|
|
}
|
|
|
|
// readMetaData gets the info if it hasn't already been fetched
|
|
func (o *Object) readMetaData() (err error) {
|
|
if o.id != "" {
|
|
return nil
|
|
}
|
|
|
|
leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(o.remote, false)
|
|
if err != nil {
|
|
if err == fs.ErrorDirNotFound {
|
|
return fs.ErrorObjectNotFound
|
|
}
|
|
return err
|
|
}
|
|
|
|
found, err := o.fs.list([]string{directoryID}, leaf, false, true, false, func(item *drive.File) bool {
|
|
if item.Name == leaf {
|
|
o.setMetaData(item)
|
|
return true
|
|
}
|
|
if !o.fs.opt.SkipGdocs {
|
|
extension, exportName, exportMimeType, _ := o.fs.findExportFormat(item)
|
|
if exportName == leaf {
|
|
o.setMetaData(item)
|
|
o.setGdocsMetaData(item, extension, exportMimeType)
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found {
|
|
return fs.ErrorObjectNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ModTime returns the modification time of the object
|
|
//
|
|
//
|
|
// It attempts to read the objects mtime and if that isn't present the
|
|
// LastModified returned in the http headers
|
|
func (o *Object) ModTime() time.Time {
|
|
err := o.readMetaData()
|
|
if err != nil {
|
|
fs.Debugf(o, "Failed to read metadata: %v", err)
|
|
return time.Now()
|
|
}
|
|
modTime, err := time.Parse(timeFormatIn, o.modifiedDate)
|
|
if err != nil {
|
|
fs.Debugf(o, "Failed to read mtime from object: %v", err)
|
|
return time.Now()
|
|
}
|
|
return modTime
|
|
}
|
|
|
|
// SetModTime sets the modification time of the drive fs object
|
|
func (o *Object) SetModTime(modTime time.Time) error {
|
|
err := o.readMetaData()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// New metadata
|
|
updateInfo := &drive.File{
|
|
ModifiedTime: modTime.Format(timeFormatOut),
|
|
}
|
|
// Set modified date
|
|
var info *drive.File
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
|
info, err = o.fs.svc.Files.Update(o.id, updateInfo).
|
|
Fields(googleapi.Field(partialFields)).
|
|
SupportsTeamDrives(o.fs.isTeamDrive).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Update info from read data
|
|
o.setMetaData(info)
|
|
return nil
|
|
}
|
|
|
|
// Storable returns a boolean as to whether this object is storable
|
|
func (o *Object) Storable() bool {
|
|
return true
|
|
}
|
|
|
|
// httpResponse gets an http.Response object for the object o.url
|
|
// using the method passed in
|
|
func (o *Object) httpResponse(method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
|
|
if o.url == "" {
|
|
return nil, nil, errors.New("forbidden to download - check sharing permission")
|
|
}
|
|
if o.isDocument {
|
|
for _, o := range options {
|
|
// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
|
|
if _, ok := o.(*fs.RangeOption); ok {
|
|
return nil, nil, errors.New("partial downloads are not supported while exporting Google Documents")
|
|
}
|
|
}
|
|
}
|
|
if o.v2Download {
|
|
f := o.fs
|
|
var v2File *drive_v2.File
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
v2File, err = o.fs.v2Svc.Files.Get(o.id).
|
|
Fields("downloadUrl").
|
|
SupportsTeamDrives(f.isTeamDrive).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err == nil {
|
|
fs.Debugf(o, "Using v2 download: %v", v2File.DownloadUrl)
|
|
o.url = v2File.DownloadUrl
|
|
o.v2Download = false
|
|
}
|
|
}
|
|
req, err = http.NewRequest(method, o.url, nil)
|
|
if err != nil {
|
|
return req, nil, err
|
|
}
|
|
fs.OpenOptionAddHTTPHeaders(req.Header, options)
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
|
res, err = o.fs.client.Do(req)
|
|
if err == nil {
|
|
err = googleapi.CheckResponse(res)
|
|
if err != nil {
|
|
_ = res.Body.Close() // ignore error
|
|
}
|
|
}
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return req, nil, err
|
|
}
|
|
return req, res, nil
|
|
}
|
|
|
|
// openFile represents an Object open for reading
|
|
type openFile struct {
|
|
o *Object // Object we are reading for
|
|
in io.ReadCloser // reading from here
|
|
bytes int64 // number of bytes read on this connection
|
|
eof bool // whether we have read end of file
|
|
errored bool // whether we have encountered an error during reading
|
|
}
|
|
|
|
// Read bytes from the object - see io.Reader
|
|
func (file *openFile) Read(p []byte) (n int, err error) {
|
|
n, err = file.in.Read(p)
|
|
file.bytes += int64(n)
|
|
if err != nil && err != io.EOF {
|
|
file.errored = true
|
|
}
|
|
if err == io.EOF {
|
|
file.eof = true
|
|
}
|
|
return
|
|
}
|
|
|
|
// Close the object and update bytes read
|
|
func (file *openFile) Close() (err error) {
|
|
// If end of file, update bytes read
|
|
if file.eof && !file.errored {
|
|
fs.Debugf(file.o, "Updating size of doc after download to %v", file.bytes)
|
|
file.o.bytes = file.bytes
|
|
}
|
|
return file.in.Close()
|
|
}
|
|
|
|
// Check it satisfies the interfaces
|
|
var _ io.ReadCloser = &openFile{}
|
|
|
|
// Checks to see if err is a googleapi.Error with of type what
|
|
func isGoogleError(err error, what string) bool {
|
|
if gerr, ok := err.(*googleapi.Error); ok {
|
|
for _, error := range gerr.Errors {
|
|
if error.Reason == what {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Open an object for read
|
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
|
_, res, err := o.httpResponse("GET", options)
|
|
if err != nil {
|
|
if isGoogleError(err, "cannotDownloadAbusiveFile") {
|
|
if o.fs.opt.AcknowledgeAbuse {
|
|
// Retry acknowledging abuse
|
|
if strings.ContainsRune(o.url, '?') {
|
|
o.url += "&"
|
|
} else {
|
|
o.url += "?"
|
|
}
|
|
o.url += "acknowledgeAbuse=true"
|
|
_, res, err = o.httpResponse("GET", options)
|
|
} else {
|
|
err = errors.Wrap(err, "Use the --drive-acknowledge-abuse flag to download this file")
|
|
}
|
|
}
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "open file failed")
|
|
}
|
|
}
|
|
// If it is a document, update the size with what we are
|
|
// reading as it can change from the HEAD in the listing to
|
|
// this GET. This stops rclone marking the transfer as
|
|
// corrupted.
|
|
if o.isDocument {
|
|
return &openFile{o: o, in: res.Body}, nil
|
|
}
|
|
return res.Body, nil
|
|
}
|
|
|
|
// Update the already existing object
|
|
//
|
|
// Copy the reader into the object updating modTime and size
|
|
//
|
|
// The new object may have been created if an error is returned
|
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
|
size := src.Size()
|
|
modTime := src.ModTime()
|
|
if o.isDocument {
|
|
return errors.New("can't update a google document")
|
|
}
|
|
updateInfo := &drive.File{
|
|
MimeType: fs.MimeType(src),
|
|
ModifiedTime: modTime.Format(timeFormatOut),
|
|
}
|
|
|
|
// Make the API request to upload metadata and file data.
|
|
var err error
|
|
var info *drive.File
|
|
if size == 0 || size < int64(o.fs.opt.UploadCutoff) {
|
|
// Don't retry, return a retry error instead
|
|
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
|
info, err = o.fs.svc.Files.Update(o.id, updateInfo).
|
|
Media(in, googleapi.ContentType("")).
|
|
Fields(googleapi.Field(partialFields)).
|
|
SupportsTeamDrives(o.fs.isTeamDrive).
|
|
KeepRevisionForever(o.fs.opt.KeepRevisionForever).
|
|
Do()
|
|
return shouldRetry(err)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
// Upload the file in chunks
|
|
info, err = o.fs.Upload(in, size, updateInfo.MimeType, o.id, updateInfo, o.remote)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
o.setMetaData(info)
|
|
return nil
|
|
}
|
|
|
|
// Remove an object
|
|
func (o *Object) Remove() error {
|
|
if o.isDocument {
|
|
return errors.New("can't delete a google document")
|
|
}
|
|
var err error
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
|
if o.fs.opt.UseTrash {
|
|
info := drive.File{
|
|
Trashed: true,
|
|
}
|
|
_, err = o.fs.svc.Files.Update(o.id, &info).
|
|
Fields("").
|
|
SupportsTeamDrives(o.fs.isTeamDrive).
|
|
Do()
|
|
} else {
|
|
err = o.fs.svc.Files.Delete(o.id).
|
|
Fields("").
|
|
SupportsTeamDrives(o.fs.isTeamDrive).
|
|
Do()
|
|
}
|
|
return shouldRetry(err)
|
|
})
|
|
return err
|
|
}
|
|
|
|
// MimeType of an Object if known, "" otherwise
|
|
func (o *Object) MimeType() string {
|
|
err := o.readMetaData()
|
|
if err != nil {
|
|
fs.Debugf(o, "Failed to read metadata: %v", err)
|
|
return ""
|
|
}
|
|
return o.mimeType
|
|
}
|
|
|
|
// ID returns the ID of the Object if known, or "" if not
|
|
func (o *Object) ID() string {
|
|
return o.id
|
|
}
|
|
|
|
// Check the interfaces are satisfied
|
|
var (
|
|
_ fs.Fs = (*Fs)(nil)
|
|
_ fs.Purger = (*Fs)(nil)
|
|
_ fs.CleanUpper = (*Fs)(nil)
|
|
_ fs.PutStreamer = (*Fs)(nil)
|
|
_ fs.Copier = (*Fs)(nil)
|
|
_ fs.Mover = (*Fs)(nil)
|
|
_ fs.DirMover = (*Fs)(nil)
|
|
_ fs.DirCacheFlusher = (*Fs)(nil)
|
|
_ fs.ChangeNotifier = (*Fs)(nil)
|
|
_ fs.PutUncheckeder = (*Fs)(nil)
|
|
_ fs.PublicLinker = (*Fs)(nil)
|
|
_ fs.ListRer = (*Fs)(nil)
|
|
_ fs.MergeDirser = (*Fs)(nil)
|
|
_ fs.Abouter = (*Fs)(nil)
|
|
_ fs.Object = (*Object)(nil)
|
|
_ fs.MimeTyper = (*Object)(nil)
|
|
_ fs.IDer = (*Object)(nil)
|
|
)
|