20c9e0cab6
This fixes the following error I encountered: ``` 2023/08/09 16:18:49 DEBUG : failed parsing fichier error: strconv.Atoi: parsing "#374": invalid syntax 2023/08/09 16:18:49 DEBUG : pacer: low level retry 1/10 (error HTTP error 403 (403 Forbidden) returned body: "{\"status\":\"KO\",\"message\":\"Flood detected: IP Locked #374\"}") ```
587 lines
14 KiB
Go
587 lines
14 KiB
Go
package fichier
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/fserrors"
|
|
"github.com/rclone/rclone/lib/rest"
|
|
)
|
|
|
|
// retryErrorCodes is a slice of error codes that we will retry
|
|
var retryErrorCodes = []int{
|
|
429, // Too Many Requests.
|
|
403, // Forbidden (may happen when request limit is exceeded)
|
|
500, // Internal Server Error
|
|
502, // Bad Gateway
|
|
503, // Service Unavailable
|
|
504, // Gateway Timeout
|
|
509, // Bandwidth Limit Exceeded
|
|
}
|
|
|
|
var errorRegex = regexp.MustCompile(`#(\d{1,3})`)
|
|
|
|
func parseFichierError(err error) int {
|
|
matches := errorRegex.FindStringSubmatch(err.Error())
|
|
if len(matches) == 0 {
|
|
return 0
|
|
}
|
|
code, err := strconv.Atoi(matches[1])
|
|
if err != nil {
|
|
fs.Debugf(nil, "failed parsing fichier error: %v", err)
|
|
return 0
|
|
}
|
|
return code
|
|
}
|
|
|
|
// shouldRetry returns a boolean as to whether this resp and err
|
|
// deserve to be retried. It returns the err as a convenience
|
|
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
|
if fserrors.ContextError(ctx, &err) {
|
|
return false, err
|
|
}
|
|
// 1Fichier uses HTTP error code 403 (Forbidden) for all kinds of errors with
|
|
// responses looking like this: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
|
|
//
|
|
// We attempt to parse the actual 1Fichier error code from this body and handle it accordingly
|
|
// Most importantly #374 (Flood detected: IP locked) which the integration tests provoke
|
|
// The list below is far from complete and should be expanded if we see any more error codes.
|
|
if err != nil {
|
|
switch parseFichierError(err) {
|
|
case 93:
|
|
return false, err // No such user
|
|
case 186:
|
|
return false, err // IP blocked?
|
|
case 374:
|
|
fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
|
|
time.Sleep(30 * time.Second)
|
|
default:
|
|
}
|
|
}
|
|
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
|
}
|
|
|
|
var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString
|
|
|
|
func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, leaf string, directoryID string, err error) {
|
|
// Create the directory for the object if it doesn't exist
|
|
leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
|
|
if err != nil {
|
|
return
|
|
}
|
|
// Temporary Object under construction
|
|
o = &Object{
|
|
fs: f,
|
|
remote: remote,
|
|
}
|
|
return o, leaf, directoryID, nil
|
|
}
|
|
|
|
func (f *Fs) readFileInfo(ctx context.Context, url string) (*File, error) {
|
|
request := FileInfoRequest{
|
|
URL: url,
|
|
}
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/file/info.cgi",
|
|
}
|
|
|
|
var file File
|
|
err := f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, &request, &file)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't read file info: %w", err)
|
|
}
|
|
|
|
return &file, err
|
|
}
|
|
|
|
// maybe do some actual validation later if necessary
|
|
func validToken(token *GetTokenResponse) bool {
|
|
return token.Status == "OK"
|
|
}
|
|
|
|
func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenResponse, error) {
|
|
request := DownloadRequest{
|
|
URL: url,
|
|
Single: 1,
|
|
Pass: f.opt.FilePassword,
|
|
}
|
|
if f.opt.CDN {
|
|
request.CDN = 1
|
|
}
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/download/get_token.cgi",
|
|
}
|
|
|
|
var token GetTokenResponse
|
|
err := f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, &request, &token)
|
|
doretry, err := shouldRetry(ctx, resp, err)
|
|
return doretry || !validToken(&token), err
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't list files: %w", err)
|
|
}
|
|
|
|
return &token, nil
|
|
}
|
|
|
|
func fileFromSharedFile(file *SharedFile) File {
|
|
return File{
|
|
URL: file.Link,
|
|
Filename: file.Filename,
|
|
Size: file.Size,
|
|
}
|
|
}
|
|
|
|
func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntries, err error) {
|
|
opts := rest.Opts{
|
|
Method: "GET",
|
|
RootURL: "https://1fichier.com/dir/",
|
|
Path: id,
|
|
Parameters: map[string][]string{"json": {"1"}},
|
|
ContentType: "application/x-www-form-urlencoded",
|
|
}
|
|
if f.opt.FolderPassword != "" {
|
|
opts.Method = "POST"
|
|
opts.Parameters = nil
|
|
opts.Body = strings.NewReader("json=1&pass=" + url.QueryEscape(f.opt.FolderPassword))
|
|
}
|
|
|
|
var sharedFiles SharedFolderResponse
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't list files: %w", err)
|
|
}
|
|
|
|
entries = make([]fs.DirEntry, len(sharedFiles))
|
|
|
|
for i, sharedFile := range sharedFiles {
|
|
entries[i] = f.newObjectFromFile(ctx, "", fileFromSharedFile(&sharedFile))
|
|
}
|
|
|
|
return entries, nil
|
|
}
|
|
|
|
func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesList, err error) {
|
|
// fs.Debugf(f, "Requesting files for dir `%s`", directoryID)
|
|
request := ListFilesRequest{
|
|
FolderID: directoryID,
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/file/ls.cgi",
|
|
}
|
|
|
|
filesList = &FilesList{}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't list files: %w", err)
|
|
}
|
|
for i := range filesList.Items {
|
|
item := &filesList.Items[i]
|
|
item.Filename = f.opt.Enc.ToStandardName(item.Filename)
|
|
}
|
|
|
|
return filesList, nil
|
|
}
|
|
|
|
func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *FoldersList, err error) {
|
|
// fs.Debugf(f, "Requesting folders for id `%s`", directoryID)
|
|
|
|
request := ListFolderRequest{
|
|
FolderID: directoryID,
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/folder/ls.cgi",
|
|
}
|
|
|
|
foldersList = &FoldersList{}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't list folders: %w", err)
|
|
}
|
|
foldersList.Name = f.opt.Enc.ToStandardName(foldersList.Name)
|
|
for i := range foldersList.SubFolders {
|
|
folder := &foldersList.SubFolders[i]
|
|
folder.Name = f.opt.Enc.ToStandardName(folder.Name)
|
|
}
|
|
|
|
// fs.Debugf(f, "Got FoldersList for id `%s`", directoryID)
|
|
|
|
return foldersList, err
|
|
}
|
|
|
|
func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
|
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
folderID, err := strconv.Atoi(directoryID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
files, err := f.listFiles(ctx, folderID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
folders, err := f.listFolders(ctx, folderID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
entries = make([]fs.DirEntry, len(files.Items)+len(folders.SubFolders))
|
|
|
|
for i, item := range files.Items {
|
|
entries[i] = f.newObjectFromFile(ctx, dir, item)
|
|
}
|
|
|
|
for i, folder := range folders.SubFolders {
|
|
createDate, err := time.Parse("2006-01-02 15:04:05", folder.CreateDate)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
fullPath := getRemote(dir, folder.Name)
|
|
folderID := strconv.Itoa(folder.ID)
|
|
|
|
entries[len(files.Items)+i] = fs.NewDir(fullPath, createDate).SetID(folderID)
|
|
|
|
// fs.Debugf(f, "Put Path `%s` for id `%d` into dircache", fullPath, folder.ID)
|
|
f.dirCache.Put(fullPath, folderID)
|
|
}
|
|
|
|
return entries, nil
|
|
}
|
|
|
|
func (f *Fs) newObjectFromFile(ctx context.Context, dir string, item File) *Object {
|
|
return &Object{
|
|
fs: f,
|
|
remote: getRemote(dir, item.Filename),
|
|
file: item,
|
|
}
|
|
}
|
|
|
|
func getRemote(dir, fileName string) string {
|
|
if dir == "" {
|
|
return fileName
|
|
}
|
|
|
|
return dir + "/" + fileName
|
|
}
|
|
|
|
func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (response *MakeFolderResponse, err error) {
|
|
name := f.opt.Enc.FromStandardName(leaf)
|
|
// fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID)
|
|
|
|
request := MakeFolderRequest{
|
|
FolderID: folderID,
|
|
Name: name,
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/folder/mkdir.cgi",
|
|
}
|
|
|
|
response = &MakeFolderResponse{}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, &request, response)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't create folder: %w", err)
|
|
}
|
|
|
|
// fs.Debugf(f, "Created Folder `%s` in id `%s`", name, directoryID)
|
|
|
|
return response, err
|
|
}
|
|
|
|
func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (response *GenericOKResponse, err error) {
|
|
// fs.Debugf(f, "Removing folder with id `%s`", directoryID)
|
|
|
|
request := &RemoveFolderRequest{
|
|
FolderID: folderID,
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/folder/rm.cgi",
|
|
}
|
|
|
|
response = &GenericOKResponse{}
|
|
var resp *http.Response
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err = f.rest.CallJSON(ctx, &opts, request, response)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't remove folder: %w", err)
|
|
}
|
|
if response.Status != "OK" {
|
|
return nil, fmt.Errorf("can't remove folder: %s", response.Message)
|
|
}
|
|
|
|
// fs.Debugf(f, "Removed Folder with id `%s`", directoryID)
|
|
|
|
return response, nil
|
|
}
|
|
|
|
func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKResponse, err error) {
|
|
request := &RemoveFileRequest{
|
|
Files: []RmFile{
|
|
{url},
|
|
},
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/file/rm.cgi",
|
|
}
|
|
|
|
response = &GenericOKResponse{}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, request, response)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't remove file: %w", err)
|
|
}
|
|
|
|
// fs.Debugf(f, "Removed file with url `%s`", url)
|
|
|
|
return response, nil
|
|
}
|
|
|
|
func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename string) (response *MoveFileResponse, err error) {
|
|
request := &MoveFileRequest{
|
|
URLs: []string{url},
|
|
FolderID: folderID,
|
|
Rename: rename,
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/file/mv.cgi",
|
|
}
|
|
|
|
response = &MoveFileResponse{}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, request, response)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
|
}
|
|
|
|
return response, nil
|
|
}
|
|
|
|
func (f *Fs) moveDir(ctx context.Context, folderID int, newLeaf string, destinationFolderID int) (response *MoveDirResponse, err error) {
|
|
request := &MoveDirRequest{
|
|
FolderID: folderID,
|
|
DestinationFolderID: destinationFolderID,
|
|
Rename: newLeaf,
|
|
// DestinationUser: destinationUser,
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/folder/mv.cgi",
|
|
}
|
|
|
|
response = &MoveDirResponse{}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, request, response)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't move dir: %w", err)
|
|
}
|
|
|
|
return response, nil
|
|
}
|
|
|
|
func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename string) (response *CopyFileResponse, err error) {
|
|
request := &CopyFileRequest{
|
|
URLs: []string{url},
|
|
FolderID: folderID,
|
|
Rename: rename,
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/file/cp.cgi",
|
|
}
|
|
|
|
response = &CopyFileResponse{}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, request, response)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
|
}
|
|
|
|
return response, nil
|
|
}
|
|
|
|
func (f *Fs) renameFile(ctx context.Context, url string, newName string) (response *RenameFileResponse, err error) {
|
|
request := &RenameFileRequest{
|
|
URLs: []RenameFileURL{
|
|
{
|
|
URL: url,
|
|
Filename: newName,
|
|
},
|
|
},
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/file/rename.cgi",
|
|
}
|
|
|
|
response = &RenameFileResponse{}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, request, response)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't rename file: %w", err)
|
|
}
|
|
|
|
return response, nil
|
|
}
|
|
|
|
func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse, err error) {
|
|
// fs.Debugf(f, "Requesting Upload node")
|
|
|
|
opts := rest.Opts{
|
|
Method: "GET",
|
|
ContentType: "application/json", // 1Fichier API is bad
|
|
Path: "/upload/get_upload_server.cgi",
|
|
}
|
|
|
|
response = &GetUploadNodeResponse{}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("didn't get an upload node: %w", err)
|
|
}
|
|
|
|
// fs.Debugf(f, "Got Upload node")
|
|
|
|
return response, err
|
|
}
|
|
|
|
func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, folderID, uploadID, node string, options ...fs.OpenOption) (response *http.Response, err error) {
|
|
// fs.Debugf(f, "Uploading File `%s`", fileName)
|
|
|
|
fileName = f.opt.Enc.FromStandardName(fileName)
|
|
|
|
if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
|
|
return nil, errors.New("invalid UploadID")
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "POST",
|
|
Path: "/upload.cgi",
|
|
Parameters: map[string][]string{
|
|
"id": {uploadID},
|
|
},
|
|
NoResponse: true,
|
|
Body: in,
|
|
ContentLength: &size,
|
|
Options: options,
|
|
MultipartContentName: "file[]",
|
|
MultipartFileName: fileName,
|
|
MultipartParams: map[string][]string{
|
|
"did": {folderID},
|
|
},
|
|
}
|
|
|
|
if node != "" {
|
|
opts.RootURL = "https://" + node
|
|
}
|
|
|
|
err = f.pacer.CallNoRetry(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, nil, nil)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't upload file: %w", err)
|
|
}
|
|
|
|
// fs.Debugf(f, "Uploaded File `%s`", fileName)
|
|
|
|
return response, err
|
|
}
|
|
|
|
func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) {
|
|
// fs.Debugf(f, "Ending File Upload `%s`", uploadID)
|
|
|
|
if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
|
|
return nil, errors.New("invalid UploadID")
|
|
}
|
|
|
|
opts := rest.Opts{
|
|
Method: "GET",
|
|
Path: "/end.pl",
|
|
RootURL: "https://" + nodeurl,
|
|
Parameters: map[string][]string{
|
|
"xid": {uploadID},
|
|
},
|
|
ExtraHeaders: map[string]string{
|
|
"JSON": "1",
|
|
},
|
|
}
|
|
|
|
response = &EndFileUploadResponse{}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
|
|
return shouldRetry(ctx, resp, err)
|
|
})
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't finish file upload: %w", err)
|
|
}
|
|
|
|
return response, err
|
|
}
|