diff --git a/backend/pikpak/api/types.go b/backend/pikpak/api/types.go index 96c16ac24..a81bda3c1 100644 --- a/backend/pikpak/api/types.go +++ b/backend/pikpak/api/types.go @@ -513,6 +513,72 @@ type RequestDecompress struct { DefaultParent bool `json:"default_parent,omitempty"` } +// ------------------------------------------------------------ authorization + +// CaptchaToken is a response to requestCaptchaToken api call +type CaptchaToken struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` // currently 300s + // API doesn't provide Expiry field and thus it should be populated from ExpiresIn on retrieval + Expiry time.Time `json:"expiry,omitempty"` + URL string `json:"url,omitempty"` // a link for users to solve captcha +} + +// expired reports whether the token is expired. +// t must be non-nil. +func (t *CaptchaToken) expired() bool { + if t.Expiry.IsZero() { + return false + } + + expiryDelta := time.Duration(10) * time.Second // same as oauth2's defaultExpiryDelta + return t.Expiry.Round(0).Add(-expiryDelta).Before(time.Now()) +} + +// Valid reports whether t is non-nil, has an AccessToken, and is not expired. +func (t *CaptchaToken) Valid() bool { + return t != nil && t.CaptchaToken != "" && !t.expired() +} + +// CaptchaTokenRequest is to request for captcha token +type CaptchaTokenRequest struct { + Action string `json:"action,omitempty"` + CaptchaToken string `json:"captcha_token,omitempty"` + ClientID string `json:"client_id,omitempty"` + DeviceID string `json:"device_id,omitempty"` + Meta *CaptchaTokenMeta `json:"meta,omitempty"` +} + +// CaptchaTokenMeta contains meta info for CaptchaTokenRequest +type CaptchaTokenMeta struct { + CaptchaSign string `json:"captcha_sign,omitempty"` + ClientVersion string `json:"client_version,omitempty"` + PackageName string `json:"package_name,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + UserID string `json:"user_id,omitempty"` // webdrive uses this instead of UserName + UserName string `json:"username,omitempty"` + Email string `json:"email,omitempty"` + PhoneNumber string `json:"phone_number,omitempty"` +} + +// Token represents oauth2 token used for pikpak which needs to be converted to be compatible with oauth2.Token +type Token struct { + TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int `json:"expires_in"` + Sub string `json:"sub"` +} + +// Expiry returns expiry from expires in, so it should be called on retrieval +// e must be non-nil. +func (e *Token) Expiry() (t time.Time) { + if v := e.ExpiresIn; v != 0 { + return time.Now().Add(time.Duration(v) * time.Second) + } + return +} + // ------------------------------------------------------------ // NOT implemented YET diff --git a/backend/pikpak/helper.go b/backend/pikpak/helper.go index fd1561d7a..ee2815d94 100644 --- a/backend/pikpak/helper.go +++ b/backend/pikpak/helper.go @@ -3,8 +3,10 @@ package pikpak import ( "bytes" "context" + "crypto/md5" "crypto/sha1" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -14,10 +16,13 @@ import ( "os" "strconv" "strings" + "sync" "time" "github.com/rclone/rclone/backend/pikpak/api" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/lib/rest" ) @@ -262,15 +267,20 @@ func (f *Fs) getGcid(ctx context.Context, src fs.ObjectInfo) (gcid string, err e if err != nil { return } + if src.Size() == 0 { + // If src is zero-length, the API will return + // Error "cid and file_size is required" (400) + // In this case, we can simply return cid == gcid + return cid, nil + } params := url.Values{} params.Set("cid", cid) params.Set("file_size", strconv.FormatInt(src.Size(), 10)) opts := rest.Opts{ - Method: "GET", - Path: "/drive/v1/resource/cid", - Parameters: params, - ExtraHeaders: map[string]string{"x-device-id": f.deviceID}, + Method: "GET", + Path: "/drive/v1/resource/cid", + Parameters: params, } info := struct { @@ -408,6 +418,8 @@ func calcCid(ctx context.Context, src fs.ObjectInfo) (cid string, err error) { return } +// ------------------------------------------------------------ authorization + // randomly generates device id used for request header 'x-device-id' // // original javascript implementation @@ -428,3 +440,206 @@ func genDeviceID() string { } return string(base) } + +var md5Salt = []string{ + "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", + "+r6CQVxjzJV6LCV", + "F", + "pFJRC", + "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", + "/750aCr4lm/Sly/c", + "RB+DT/gZCrbV", + "", + "CyLsf7hdkIRxRm215hl", + "7xHvLi2tOYP0Y92b", + "ZGTXXxu8E/MIWaEDB+Sm/", + "1UI3", + "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", + "ihtqpG6FMt65+Xk+tWUH2", + "NhXXU9rg4XXdzo7u5o", +} + +func md5Sum(text string) string { + hash := md5.Sum([]byte(text)) + return hex.EncodeToString(hash[:]) +} + +func calcCaptchaSign(deviceID string) (timestamp, sign string) { + timestamp = fmt.Sprint(time.Now().UnixMilli()) + str := fmt.Sprint(clientID, clientVersion, packageName, deviceID, timestamp) + for _, salt := range md5Salt { + str = md5Sum(str + salt) + } + sign = "1." + str + return +} + +func newCaptchaTokenRequest(action, oldToken string, opt *Options) (req *api.CaptchaTokenRequest) { + req = &api.CaptchaTokenRequest{ + Action: action, + CaptchaToken: oldToken, // can be empty initially + ClientID: clientID, + DeviceID: opt.DeviceID, + Meta: new(api.CaptchaTokenMeta), + } + switch action { + case "POST:/v1/auth/signin": + req.Meta.UserName = opt.Username + default: + timestamp, captchaSign := calcCaptchaSign(opt.DeviceID) + req.Meta.CaptchaSign = captchaSign + req.Meta.Timestamp = timestamp + req.Meta.ClientVersion = clientVersion + req.Meta.PackageName = packageName + req.Meta.UserID = opt.UserID + } + return +} + +// CaptchaTokenSource stores updated captcha tokens in the config file +type CaptchaTokenSource struct { + mu sync.Mutex + m configmap.Mapper + opt *Options + token *api.CaptchaToken + ctx context.Context + rst *pikpakClient +} + +// initialize CaptchaTokenSource from rclone.conf if possible +func newCaptchaTokenSource(ctx context.Context, opt *Options, m configmap.Mapper) *CaptchaTokenSource { + token := new(api.CaptchaToken) + tokenString, ok := m.Get("captcha_token") + if !ok || tokenString == "" { + fs.Debugf(nil, "failed to read captcha token out of config file") + } else { + if err := json.Unmarshal([]byte(tokenString), token); err != nil { + fs.Debugf(nil, "failed to parse captcha token out of config file: %v", err) + } + } + return &CaptchaTokenSource{ + m: m, + opt: opt, + token: token, + ctx: ctx, + rst: newPikpakClient(getClient(ctx, opt), opt), + } +} + +// requestToken retrieves captcha token from API +func (cts *CaptchaTokenSource) requestToken(ctx context.Context, req *api.CaptchaTokenRequest) (err error) { + opts := rest.Opts{ + Method: "POST", + RootURL: "https://user.mypikpak.com/v1/shield/captcha/init", + } + var info *api.CaptchaToken + _, err = cts.rst.CallJSON(ctx, &opts, &req, &info) + if err == nil && info.ExpiresIn != 0 { + // populate to Expiry + info.Expiry = time.Now().Add(time.Duration(info.ExpiresIn) * time.Second) + cts.token = info // update with a new one + } + return +} + +func (cts *CaptchaTokenSource) refreshToken(opts *rest.Opts) (string, error) { + oldToken := "" + if cts.token != nil { + oldToken = cts.token.CaptchaToken + } + action := "GET:/drive/v1/about" + if opts.RootURL == "" && opts.Path != "" { + action = fmt.Sprintf("%s:%s", opts.Method, opts.Path) + } else if u, err := url.Parse(opts.RootURL); err == nil { + action = fmt.Sprintf("%s:%s", opts.Method, u.Path) + } + req := newCaptchaTokenRequest(action, oldToken, cts.opt) + if err := cts.requestToken(cts.ctx, req); err != nil { + return "", fmt.Errorf("failed to retrieve captcha token from api: %w", err) + } + + // put it into rclone.conf + tokenBytes, err := json.Marshal(cts.token) + if err != nil { + return "", fmt.Errorf("failed to marshal captcha token: %w", err) + } + cts.m.Set("captcha_token", string(tokenBytes)) + return cts.token.CaptchaToken, nil +} + +// Invalidate resets existing captcha token for a forced refresh +func (cts *CaptchaTokenSource) Invalidate() { + cts.mu.Lock() + cts.token.CaptchaToken = "" + cts.mu.Unlock() +} + +// Token returns a valid captcha token +func (cts *CaptchaTokenSource) Token(opts *rest.Opts) (string, error) { + cts.mu.Lock() + defer cts.mu.Unlock() + if cts.token.Valid() { + return cts.token.CaptchaToken, nil + } + return cts.refreshToken(opts) +} + +// pikpakClient wraps rest.Client with a handle of captcha token +type pikpakClient struct { + opt *Options + client *rest.Client + captcha *CaptchaTokenSource +} + +// newPikpakClient takes an (oauth) http.Client and makes a new api instance for pikpak with +// * error handler +// * root url +// * default headers +func newPikpakClient(c *http.Client, opt *Options) *pikpakClient { + client := rest.NewClient(c).SetErrorHandler(errorHandler).SetRoot(rootURL) + for key, val := range map[string]string{ + "Referer": "https://mypikpak.com/", + "x-client-id": clientID, + "x-client-version": clientVersion, + "x-device-id": opt.DeviceID, + // "x-device-model": "firefox%2F129.0", + // "x-device-name": "PC-Firefox", + // "x-device-sign": fmt.Sprintf("wdi10.%sxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", opt.DeviceID), + // "x-net-work-type": "NONE", + // "x-os-version": "Win32", + // "x-platform-version": "1", + // "x-protocol-version": "301", + // "x-provider-name": "NONE", + // "x-sdk-version": "8.0.3", + } { + client.SetHeader(key, val) + } + return &pikpakClient{ + client: client, + opt: opt, + } +} + +// This should be called right after pikpakClient initialized +func (c *pikpakClient) SetCaptchaTokener(ctx context.Context, m configmap.Mapper) *pikpakClient { + c.captcha = newCaptchaTokenSource(ctx, c.opt, m) + return c +} + +func (c *pikpakClient) CallJSON(ctx context.Context, opts *rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) { + if c.captcha != nil { + token, err := c.captcha.Token(opts) + if err != nil || token == "" { + return nil, fserrors.FatalError(fmt.Errorf("couldn't get captcha token: %v", err)) + } + if opts.ExtraHeaders == nil { + opts.ExtraHeaders = make(map[string]string) + } + opts.ExtraHeaders["x-captcha-token"] = token + } + return c.client.CallJSON(ctx, opts, request, response) +} + +func (c *pikpakClient) Call(ctx context.Context, opts *rest.Opts) (resp *http.Response, err error) { + return c.client.Call(ctx, opts) +} diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.go index 1a5d82a33..25c27f1c4 100644 --- a/backend/pikpak/pikpak.go +++ b/backend/pikpak/pikpak.go @@ -23,6 +23,7 @@ package pikpak import ( "bytes" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -51,6 +52,7 @@ import ( "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/dircache" @@ -64,15 +66,17 @@ import ( // Constants const ( - rcloneClientID = "YNxT9w7GMdWvEOKa" - rcloneEncryptedClientSecret = "aqrmB6M1YJ1DWCBxVxFSjFo7wzWEky494YMmkqgAl1do1WKOe2E" - minSleep = 100 * time.Millisecond - maxSleep = 2 * time.Second - taskWaitTime = 500 * time.Millisecond - decayConstant = 2 // bigger for slower decay, exponential - rootURL = "https://api-drive.mypikpak.com" - minChunkSize = fs.SizeSuffix(manager.MinUploadPartSize) - defaultUploadConcurrency = manager.DefaultUploadConcurrency + clientID = "YUMx5nI8ZU8Ap8pm" + clientVersion = "2.0.0" + packageName = "mypikpak.com" + defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0" + minSleep = 100 * time.Millisecond + maxSleep = 2 * time.Second + taskWaitTime = 500 * time.Millisecond + decayConstant = 2 // bigger for slower decay, exponential + rootURL = "https://api-drive.mypikpak.com" + minChunkSize = fs.SizeSuffix(manager.MinUploadPartSize) + defaultUploadConcurrency = manager.DefaultUploadConcurrency ) // Globals @@ -85,43 +89,53 @@ var ( TokenURL: "https://user.mypikpak.com/v1/auth/token", AuthStyle: oauth2.AuthStyleInParams, }, - ClientID: rcloneClientID, - ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), - RedirectURL: oauthutil.RedirectURL, + ClientID: clientID, + RedirectURL: oauthutil.RedirectURL, } ) -// Returns OAuthOptions modified for pikpak -func pikpakOAuthOptions() []fs.Option { - opts := []fs.Option{} - for _, opt := range oauthutil.SharedOptions { - if opt.Name == config.ConfigClientID { - opt.Advanced = true - } else if opt.Name == config.ConfigClientSecret { - opt.Advanced = true - } - opts = append(opts, opt) - } - return opts -} - // pikpakAutorize retrieves OAuth token using user/pass and save it to rclone.conf func pikpakAuthorize(ctx context.Context, opt *Options, name string, m configmap.Mapper) error { - // override default client id/secret - if id, ok := m.Get("client_id"); ok && id != "" { - oauthConfig.ClientID = id - } - if secret, ok := m.Get("client_secret"); ok && secret != "" { - oauthConfig.ClientSecret = secret + if opt.Username == "" { + return errors.New("no username") } pass, err := obscure.Reveal(opt.Password) if err != nil { return fmt.Errorf("failed to decode password - did you obscure it?: %w", err) } - t, err := oauthConfig.PasswordCredentialsToken(ctx, opt.Username, pass) + // new device id if necessary + if len(opt.DeviceID) != 32 { + opt.DeviceID = genDeviceID() + m.Set("device_id", opt.DeviceID) + fs.Infof(nil, "Using new device id %q", opt.DeviceID) + } + opts := rest.Opts{ + Method: "POST", + RootURL: "https://user.mypikpak.com/v1/auth/signin", + } + req := map[string]string{ + "username": opt.Username, + "password": pass, + "client_id": clientID, + } + var token api.Token + rst := newPikpakClient(getClient(ctx, opt), opt).SetCaptchaTokener(ctx, m) + _, err = rst.CallJSON(ctx, &opts, req, &token) + if apiErr, ok := err.(*api.Error); ok { + if apiErr.Reason == "captcha_invalid" && apiErr.Code == 4002 { + rst.captcha.Invalidate() + _, err = rst.CallJSON(ctx, &opts, req, &token) + } + } if err != nil { return fmt.Errorf("failed to retrieve token using username/password: %w", err) } + t := &oauth2.Token{ + AccessToken: token.AccessToken, + TokenType: token.TokenType, + RefreshToken: token.RefreshToken, + Expiry: token.Expiry(), + } return oauthutil.PutToken(name, m, t, false) } @@ -160,7 +174,7 @@ func init() { } return nil, fmt.Errorf("unknown state %q", config.State) }, - Options: append(pikpakOAuthOptions(), []fs.Option{{ + Options: []fs.Option{{ Name: "user", Help: "Pikpak username.", Required: true, @@ -170,6 +184,18 @@ func init() { Help: "Pikpak password.", Required: true, IsPassword: true, + }, { + Name: "device_id", + Help: "Device ID used for authorization.", + Advanced: true, + Sensitive: true, + }, { + Name: "user_agent", + Default: defaultUserAgent, + Advanced: true, + Help: fmt.Sprintf(`HTTP user agent for pikpak. + +Defaults to "%s" or "--pikpak-user-agent" provided on command line.`, defaultUserAgent), }, { Name: "root_folder_id", Help: `ID of the root folder. @@ -248,7 +274,7 @@ this may help to speed up the transfers.`, encoder.EncodeRightSpace | encoder.EncodeRightPeriod | encoder.EncodeInvalidUtf8), - }}...), + }}, }) } @@ -256,6 +282,9 @@ this may help to speed up the transfers.`, type Options struct { Username string `config:"user"` Password string `config:"pass"` + UserID string `config:"user_id"` // only available during runtime + DeviceID string `config:"device_id"` + UserAgent string `config:"user_agent"` RootFolderID string `config:"root_folder_id"` UseTrash bool `config:"use_trash"` TrashedOnly bool `config:"trashed_only"` @@ -271,11 +300,10 @@ type Fs struct { root string // the path we are working on opt Options // parsed options features *fs.Features // optional features - rst *rest.Client // the connection to the server + rst *pikpakClient // the connection to the server dirCache *dircache.DirCache // Map of directory path to directory id pacer *fs.Pacer // pacer for API calls rootFolderID string // the id of the root folder - deviceID string // device id used for api requests client *http.Client // authorized client m configmap.Mapper tokenMu *sync.Mutex // when renewing tokens @@ -429,6 +457,12 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b } else if apiErr.Reason == "file_space_not_enough" { // "file_space_not_enough" (8): Storage space is not enough return false, fserrors.FatalError(err) + } else if apiErr.Reason == "captcha_invalid" && apiErr.Code == 9 { + // "captcha_invalid" (9): Verification code is invalid + // This error occurred on the POST:/drive/v1/files endpoint + // when a zero-byte file was uploaded with an invalid captcha token + f.rst.captcha.Invalidate() + return true, err } } @@ -452,13 +486,36 @@ func errorHandler(resp *http.Response) error { return errResponse } +// getClient makes an http client according to the options +func getClient(ctx context.Context, opt *Options) *http.Client { + // Override few config settings and create a client + newCtx, ci := fs.AddConfig(ctx) + ci.UserAgent = opt.UserAgent + return fshttp.NewClient(newCtx) +} + // newClientWithPacer sets a new http/rest client with a pacer to Fs func (f *Fs) newClientWithPacer(ctx context.Context) (err error) { - f.client, _, err = oauthutil.NewClient(ctx, f.name, f.m, oauthConfig) + var ts *oauthutil.TokenSource + f.client, ts, err = oauthutil.NewClientWithBaseClient(ctx, f.name, f.m, oauthConfig, getClient(ctx, &f.opt)) if err != nil { return fmt.Errorf("failed to create oauth client: %w", err) } - f.rst = rest.NewClient(f.client).SetRoot(rootURL).SetErrorHandler(errorHandler) + token, err := ts.Token() + if err != nil { + return err + } + // parse user_id from oauth access token for later use + if parts := strings.Split(token.AccessToken, "."); len(parts) > 1 { + jsonStr, _ := base64.URLEncoding.DecodeString(parts[1] + "===") + info := struct { + UserID string `json:"sub,omitempty"` + }{} + if jsonErr := json.Unmarshal(jsonStr, &info); jsonErr == nil { + f.opt.UserID = info.UserID + } + } + f.rst = newPikpakClient(f.client, &f.opt).SetCaptchaTokener(ctx, f.m) f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))) return nil } @@ -491,10 +548,19 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err CanHaveEmptyDirectories: true, // can have empty directories NoMultiThreading: true, // can't have multiple threads downloading }).Fill(ctx, f) - f.deviceID = genDeviceID() + + // new device id if necessary + if len(f.opt.DeviceID) != 32 { + f.opt.DeviceID = genDeviceID() + m.Set("device_id", f.opt.DeviceID) + fs.Infof(nil, "Using new device id %q", f.opt.DeviceID) + } if err := f.newClientWithPacer(ctx); err != nil { - return nil, err + // re-authorize if necessary + if strings.Contains(err.Error(), "invalid_grant") { + return f, f.reAuthorize(ctx) + } } return f, nil