KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

plugins: restructure and add tests for pluginsctl/* calls

Browse source
This commit is contained in:
Chaitanya Bankanhal 2020-08-20 23:31:38 +05:30 committed by Nick Craig-Wood
parent cf68e61f40
commit 09b79679cd
5 changed files with 547 additions and 404 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
fs/rc/rcserver/rcserver.go
View file

@ -6,7 +6,6 @@ import (
"encoding/json" "encoding/json"
"flag" "flag"
"fmt" "fmt"
"github.com/rclone/rclone/fs/rc/webgui"
"log" "log"
"mime" "mime"
"net/http" "net/http"
@ -18,6 +17,8 @@ import (
"sync" "sync"
"time" "time"
"github.com/rclone/rclone/fs/rc/webgui"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
@ -368,7 +369,6 @@ func (s *Server) serveRemote(w http.ResponseWriter, r *http.Request, path string
// Match URLS of the form [fs]/remote // Match URLS of the form [fs]/remote
var fsMatch = regexp.MustCompile(`^\[(.*?)\](.*)$`) var fsMatch = regexp.MustCompile(`^\[(.*?)\](.*)$`)
var referrerPathReg = regexp.MustCompile("^(https?)://(.+):([0-9]+)?/(.*)$")
func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string) { func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string) {
// Look to see if this has an fs in the path // Look to see if this has an fs in the path
@ -397,21 +397,9 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
return return
} }
return return
} else if s.opt.WebUI { } else if s.opt.WebUI && webgui.ServePluginWithReferrerOK(w, r, path) {
referrer := r.Referer()
referrerPathMatch := referrerPathReg.FindStringSubmatch(referrer)
if referrerPathMatch != nil {
referrerPluginMatch := webgui.PluginsMatch.FindStringSubmatch(referrerPathMatch[4])
if referrerPluginMatch != nil {
path = fmt.Sprintf("/plugins/%s/%s/%s", referrerPluginMatch[1], referrerPluginMatch[2], path)
http.Redirect(w, r, path, http.StatusMovedPermanently)
//s.pluginsHandler.ServeHTTP(w, r)
return return
} }
}
}
// Serve the files // Serve the files
r.URL.Path = "/" + path r.URL.Path = "/" + path
s.files.ServeHTTP(w, r) s.files.ServeHTTP(w, r)

456
fs/rc/webgui/plugins.go
View file

@ -1,24 +1,22 @@
package webgui package webgui
import ( import (
"context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/rc"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/http/httputil" "net/http/httputil"
"net/url" "net/url"
"os"
"path/filepath" "path/filepath"
"regexp" "regexp"
"strings" "strings"
"sync" "sync"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
) )
// PackageJSON is the structure of package.json of a plugin
type PackageJSON struct { type PackageJSON struct {
Name string `json:"name"` Name string `json:"name"`
Version string `json:"version"` Version string `json:"version"`
@ -36,23 +34,33 @@ type PackageJSON struct {
Bugs struct { Bugs struct {
URL string `json:"url"` URL string `json:"url"`
} `json:"bugs"` } `json:"bugs"`
//RcloneHandlesType []string `json:"rcloneHandlesType"`
Rclone RcloneConfig `json:"rclone"` Rclone RcloneConfig `json:"rclone"`
} }
// RcloneConfig represents the rclone specific config
type RcloneConfig struct { type RcloneConfig struct {
HandlesType []string `json:"handlesType"` HandlesType []string `json:"handlesType"`
PluginType string `json:"pluginType"` PluginType string `json:"pluginType"`
RedirectReferrer bool `json:"redirectReferrer"`
Test bool `json:"-"`
}
func (r *PackageJSON) isTesting() bool {
return r.Rclone.Test
} }
var ( var (
loadedTestPlugins *Plugins //loadedTestPlugins *Plugins
cachePath string cachePath string
PluginsPath string
pluginsConfigPath string
loadedPlugins *Plugins loadedPlugins *Plugins
pluginsProxy = &httputil.ReverseProxy{} pluginsProxy = &httputil.ReverseProxy{}
// PluginsMatch is used for matching author and plugin name in the url path
PluginsMatch = regexp.MustCompile(`^plugins\/([^\/]*)\/([^\/\?]+)[\/]?(.*)$`)
// PluginsPath is the base path where webgui plugins are stored
PluginsPath string
pluginsConfigPath string
availablePluginsJSONPath = "availablePlugins.json"
) )
func init() { func init() {
@ -60,19 +68,14 @@ func init() {
PluginsPath = filepath.Join(cachePath, "plugins") PluginsPath = filepath.Join(cachePath, "plugins")
pluginsConfigPath = filepath.Join(PluginsPath, "config") pluginsConfigPath = filepath.Join(PluginsPath, "config")
loadedPlugins = newPlugins("availablePlugins.json") loadedPlugins = newPlugins(availablePluginsJSONPath)
err := loadedPlugins.readFromFile() err := loadedPlugins.readFromFile()
if err != nil { if err != nil {
fs.Errorf(nil, "error reading available plugins", err) fs.Errorf(nil, "error reading available plugins: %v", err)
}
loadedTestPlugins = newPlugins("testPlugins.json")
err = loadedTestPlugins.readFromFile()
if err != nil {
fs.Errorf(nil, "error reading test plugins", err)
} }
} }
// Plugins represents the structure how plugins are saved onto disk
type Plugins struct { type Plugins struct {
mutex sync.Mutex mutex sync.Mutex
LoadedPlugins map[string]PackageJSON `json:"loadedPlugins"` LoadedPlugins map[string]PackageJSON `json:"loadedPlugins"`
@ -93,8 +96,8 @@ func (p *Plugins) readFromFile() (err error) {
if err != nil { if err != nil {
return err return err
} }
availablePluginsJson := filepath.Join(pluginsConfigPath, p.fileName) availablePluginsJSON := filepath.Join(pluginsConfigPath, p.fileName)
data, err := ioutil.ReadFile(availablePluginsJson) data, err := ioutil.ReadFile(availablePluginsJSON)
if err != nil { if err != nil {
// create a file ? // create a file ?
} }
@ -105,19 +108,19 @@ func (p *Plugins) readFromFile() (err error) {
return nil return nil
} }
func (p *Plugins) addPlugin(pluginName string, packageJsonPath string) (err error) { func (p *Plugins) addPlugin(pluginName string, packageJSONPath string) (err error) {
p.mutex.Lock() p.mutex.Lock()
defer p.mutex.Unlock() defer p.mutex.Unlock()
data, err := ioutil.ReadFile(packageJsonPath) data, err := ioutil.ReadFile(packageJSONPath)
if err != nil { if err != nil {
return err return err
} }
var pkgJson = PackageJSON{} var pkgJSON = PackageJSON{}
err = json.Unmarshal(data, &pkgJson) err = json.Unmarshal(data, &pkgJSON)
if err != nil { if err != nil {
return err return err
} }
p.LoadedPlugins[pluginName] = pkgJson p.LoadedPlugins[pluginName] = pkgJSON
err = p.writeToFile() err = p.writeToFile()
if err != nil { if err != nil {
@ -135,15 +138,16 @@ func (p *Plugins) addTestPlugin(pluginName string, testURL string, handlesType [
return err return err
} }
var pkgJson = PackageJSON{ var pkgJSON = PackageJSON{
Name: pluginName, Name: pluginName,
TestURL: testURL, TestURL: testURL,
Rclone: RcloneConfig{ Rclone: RcloneConfig{
HandlesType: handlesType, HandlesType: handlesType,
Test: true,
}, },
} }
p.LoadedPlugins[pluginName] = pkgJson p.LoadedPlugins[pluginName] = pkgJSON
err = p.writeToFile() err = p.writeToFile()
if err != nil { if err != nil {
@ -156,11 +160,11 @@ func (p *Plugins) addTestPlugin(pluginName string, testURL string, handlesType [
func (p *Plugins) writeToFile() (err error) { func (p *Plugins) writeToFile() (err error) {
//p.mutex.Lock() //p.mutex.Lock()
//defer p.mutex.Unlock() //defer p.mutex.Unlock()
availablePluginsJson := filepath.Join(pluginsConfigPath, p.fileName) availablePluginsJSON := filepath.Join(pluginsConfigPath, p.fileName)
file, err := json.MarshalIndent(p, "", " ") file, err := json.MarshalIndent(p, "", " ")
err = ioutil.WriteFile(availablePluginsJson, file, 0755) err = ioutil.WriteFile(availablePluginsJSON, file, 0755)
if err != nil { if err != nil {
fs.Logf(nil, "%s", err) fs.Logf(nil, "%s", err)
} }
@ -177,7 +181,7 @@ func (p *Plugins) removePlugin(name string) (err error) {
_, ok := p.LoadedPlugins[name] _, ok := p.LoadedPlugins[name]
if !ok { if !ok {
return errors.New(fmt.Sprintf("plugin %s not loaded", name)) return fmt.Errorf("plugin %s not loaded", name)
} }
delete(p.LoadedPlugins, name) delete(p.LoadedPlugins, name)
@ -188,219 +192,18 @@ func (p *Plugins) removePlugin(name string) (err error) {
return nil return nil
} }
// GetPluginByName returns the plugin object for the key (author/plugin-name)
func (p *Plugins) GetPluginByName(name string) (out *PackageJSON, err error) { func (p *Plugins) GetPluginByName(name string) (out *PackageJSON, err error) {
p.mutex.Lock() p.mutex.Lock()
defer p.mutex.Unlock() defer p.mutex.Unlock()
po, ok := p.LoadedPlugins[name] po, ok := p.LoadedPlugins[name]
if !ok { if !ok {
return nil, errors.New(fmt.Sprintf("plugin %s not loaded", name)) return nil, fmt.Errorf("plugin %s not loaded", name)
} }
return &po, nil return &po, nil
} }
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/addTestPlugin",
AuthRequired: true,
Fn: rcAddTestPlugin,
Title: "Show current mount points",
Help: `This shows currently mounted points, which can be used for performing an unmount
This takes no parameters and returns
- mountPoints: list of current mount points
Eg
rclone rc pluginsctl/addTestPlugin
`,
})
}
func rcAddTestPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
name, err := in.GetString("name")
if err != nil {
return nil, err
}
loadUrl, err := in.GetString("loadUrl")
if err != nil {
return nil, err
}
var handlesTypes []string
err = in.GetStructMissingOK("handlesTypes", &handlesTypes)
if err != nil {
return nil, err
}
err = loadedTestPlugins.addTestPlugin(name, loadUrl, handlesTypes)
if err != nil {
return nil, err
}
return nil, nil
}
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/listTestPlugins",
AuthRequired: true,
Fn: rcGetLoadedPlugins,
Title: "Show current mount points",
Help: `This shows currently mounted points, which can be used for performing an unmount
This takes no parameters and returns
- mountPoints: list of current mount points
Eg
rclone rc pluginsctl/listTestPlugins
`,
})
}
func rcGetLoadedPlugins(_ context.Context, in rc.Params) (out rc.Params, err error) {
return rc.Params{
"loadedTestPlugins": loadedTestPlugins.LoadedPlugins,
}, nil
}
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/removeTestPlugin",
AuthRequired: true,
Fn: rcRemoveTestPlugin,
Title: "Show current mount points",
Help: `This shows currently mounted points, which can be used for performing an unmount
This takes no parameters and returns
- mountPoints: list of current mount points
Eg
rclone rc pluginsctl/removeTestPlugin
`,
})
}
func rcRemoveTestPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
name, err := in.GetString("name")
if err != nil {
return nil, err
}
err = loadedTestPlugins.removePlugin(name)
if err != nil {
return nil, err
}
return nil, nil
}
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/addPlugin",
AuthRequired: true,
Fn: rcAddPlugin,
Title: "Show current mount points",
Help: `This shows currently mounted points, which can be used for performing an unmount
This takes no parameters and returns
- mountPoints: list of current mount points
Eg
rclone rc pluginsctl/addPlugin
`,
})
}
func rcAddPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
pluginUrl, err := in.GetString("url")
if err != nil {
return nil, err
}
author, repoName, repoBranch, err := getAuthorRepoBranchGithub(pluginUrl)
if err != nil {
return nil, err
}
branch, err := in.GetString("branch")
if err != nil || branch == "" {
branch = repoBranch
}
version, err := in.GetString("version")
if err != nil || version == "" {
version = "latest"
}
err = CreatePathIfNotExist(PluginsPath)
if err != nil {
return nil, err
}
// fetch and package.json
// https://raw.githubusercontent.com/rclone/rclone-webui-react/master/package.json
pluginID := fmt.Sprintf("%s/%s", author, repoName)
currentPluginPath := filepath.Join(PluginsPath, pluginID)
err = CreatePathIfNotExist(currentPluginPath)
if err != nil {
return nil, err
}
packageJsonUrl := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/package.json", author, repoName, branch)
packageJsonFilePath := filepath.Join(currentPluginPath, "package.json")
err = DownloadFile(packageJsonFilePath, packageJsonUrl)
if err != nil {
return nil, err
}
// register in plugins
// download release and save in plugins/<author>/repo-name/app
// https://api.github.com/repos/rclone/rclone-webui-react/releases/latest
releaseUrl, tag, _, err := GetLatestReleaseURL(fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/%s", author, repoName, version))
zipName := tag + ".zip"
zipPath := filepath.Join(currentPluginPath, zipName)
err = DownloadFile(zipPath, releaseUrl)
if err != nil {
return nil, err
}
extractPath := filepath.Join(currentPluginPath, "app")
err = CreatePathIfNotExist(extractPath)
if err != nil {
return nil, err
}
err = os.RemoveAll(extractPath)
if err != nil {
fs.Logf(nil, "No previous downloads to remove")
}
fs.Logf(nil, "Unzipping plugin binary")
err = Unzip(zipPath, extractPath)
if err != nil {
return nil, err
}
err = loadedPlugins.addPlugin(pluginID, packageJsonFilePath)
if err != nil {
return nil, err
}
return nil, nil
}
// getAuthorRepoBranchGithub gives author, repoName and branch from a github.com url // getAuthorRepoBranchGithub gives author, repoName and branch from a github.com url
// url examples: // url examples:
// https://github.com/rclone/rclone-webui-react/ // https://github.com/rclone/rclone-webui-react/
@ -409,14 +212,14 @@ func rcAddPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
// github.com/rclone/rclone-webui-react // github.com/rclone/rclone-webui-react
// //
func getAuthorRepoBranchGithub(url string) (author string, repoName string, branch string, err error) { func getAuthorRepoBranchGithub(url string) (author string, repoName string, branch string, err error) {
repoUrl := url repoURL := url
repoUrl = strings.Replace(repoUrl, "https://", "", 1) repoURL = strings.Replace(repoURL, "https://", "", 1)
repoUrl = strings.Replace(repoUrl, "http://", "", 1) repoURL = strings.Replace(repoURL, "http://", "", 1)
urlSplits := strings.Split(repoUrl, "/") urlSplits := strings.Split(repoURL, "/")
if len(urlSplits) < 3 || len(urlSplits) > 5 || urlSplits[0] != "github.com" { if len(urlSplits) < 3 || len(urlSplits) > 5 || urlSplits[0] != "github.com" {
return "", "", "", errors.New(fmt.Sprintf("Invalid github url: %s", url)) return "", "", "", fmt.Errorf("invalid github url: %s", url)
} }
// get branch name // get branch name
@ -427,94 +230,6 @@ func getAuthorRepoBranchGithub(url string) (author string, repoName string, bran
return urlSplits[1], urlSplits[2], "master", nil return urlSplits[1], urlSplits[2], "master", nil
} }
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/listPlugins",
AuthRequired: true,
Fn: rcGetPlugins,
Title: "Get the list of currently loaded plugins",
Help: `This allows you to get the currently enabled plugins and their details.
This takes no parameters and returns
- loadedPlugins: list of current production plugins
- testPlugins: list of temporarily loaded development plugins, usually running on a different server.
Eg
rclone rc pluginsctl/listPlugins
`,
})
}
func rcGetPlugins(_ context.Context, in rc.Params) (out rc.Params, err error) {
err = loadedPlugins.readFromFile()
if err != nil {
return nil, err
}
err = loadedTestPlugins.readFromFile()
if err != nil {
return nil, err
}
return rc.Params{
"loadedPlugins": loadedPlugins.LoadedPlugins,
"loadedTestPlugins": loadedTestPlugins.LoadedPlugins,
}, nil
}
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/removePlugin",
AuthRequired: true,
Fn: rcRemovePlugin,
Title: "Get the list of currently loaded plugins",
Help: `This allows you to get the currently enabled plugins and their details.
This takes parameters
- name: name of the plugin in the format <author>/<plugin_name>
Eg
rclone rc pluginsctl/removePlugin name=rclone/video-plugin
`,
})
}
func rcRemovePlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
name, err := in.GetString("name")
if err != nil {
return nil, err
}
err = loadedPlugins.removePlugin(name)
if err != nil {
return nil, err
}
return nil, nil
}
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/getPluginsForType",
AuthRequired: true,
Fn: rcGetPluginsForType,
Title: "Get the list of currently loaded plugins",
Help: `This allows you to get the currently enabled plugins and their details.
This takes no parameters and returns
- loadedPlugins: list of current production plugins
- testPlugins: list of temporarily loaded development plugins, usually running on a different server.
Eg
rclone rc pluginsctl/getPlugins
`,
})
}
func filterPlugins(plugins *Plugins, compare func(packageJSON *PackageJSON) bool) map[string]PackageJSON { func filterPlugins(plugins *Plugins, compare func(packageJSON *PackageJSON) bool) map[string]PackageJSON {
output := map[string]PackageJSON{} output := map[string]PackageJSON{}
@ -527,58 +242,7 @@ func filterPlugins(plugins *Plugins, compare func(packageJSON *PackageJSON) bool
return output return output
} }
func rcGetPluginsForType(_ context.Context, in rc.Params) (out rc.Params, err error) { // getDirectorForProxy is a helper function for reverse proxy of test plugins
handlesType, err := in.GetString("type")
if err != nil {
handlesType = ""
}
pluginType, err := in.GetString("pluginType")
if err != nil {
pluginType = ""
}
var loadedPluginsResult map[string]PackageJSON
var loadedTestPluginsResult map[string]PackageJSON
if pluginType == "" || pluginType == "FileHandler" {
loadedPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool {
for i := range packageJSON.Rclone.HandlesType {
if packageJSON.Rclone.HandlesType[i] == handlesType {
return true
}
}
return false
})
loadedTestPluginsResult = filterPlugins(loadedTestPlugins, func(packageJSON *PackageJSON) bool {
for i := range packageJSON.Rclone.HandlesType {
if packageJSON.Rclone.HandlesType[i] == handlesType {
return true
}
}
return false
})
} else {
loadedPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool {
return packageJSON.Rclone.PluginType == pluginType
})
loadedTestPluginsResult = filterPlugins(loadedTestPlugins, func(packageJSON *PackageJSON) bool {
return packageJSON.Rclone.PluginType == pluginType
})
}
return rc.Params{
"loadedPlugins": loadedPluginsResult,
"testPlugins": loadedTestPluginsResult,
}, nil
}
var PluginsMatch = regexp.MustCompile(`^plugins\/([^\/]*)\/([^\/\?]+)[\/]?(.*)$`)
func getDirectorForProxy(origin *url.URL) func(req *http.Request) { func getDirectorForProxy(origin *url.URL) func(req *http.Request) {
return func(req *http.Request) { return func(req *http.Request) {
req.Header.Add("X-Forwarded-Host", req.Host) req.Header.Add("X-Forwarded-Host", req.Host)
@ -589,12 +253,15 @@ func getDirectorForProxy(origin *url.URL) func(req *http.Request) {
} }
} }
// ServePluginOK checks the plugin url and uses reverse proxy to allow redirection for content not being served by rclone
func ServePluginOK(w http.ResponseWriter, r *http.Request, pluginsMatchResult []string) (ok bool) { func ServePluginOK(w http.ResponseWriter, r *http.Request, pluginsMatchResult []string) (ok bool) {
testPlugin, err := loadedTestPlugins.GetPluginByName(fmt.Sprintf("%s/%s", pluginsMatchResult[1], pluginsMatchResult[2])) testPlugin, err := loadedPlugins.GetPluginByName(fmt.Sprintf("%s/%s", pluginsMatchResult[1], pluginsMatchResult[2]))
if err != nil { if err != nil {
return false return false
} }
if !testPlugin.Rclone.Test {
return false
}
origin, _ := url.Parse(fmt.Sprintf("%s/%s", testPlugin.TestURL, pluginsMatchResult[3])) origin, _ := url.Parse(fmt.Sprintf("%s/%s", testPlugin.TestURL, pluginsMatchResult[3]))
director := getDirectorForProxy(origin) director := getDirectorForProxy(origin)
@ -603,3 +270,30 @@ func ServePluginOK(w http.ResponseWriter, r *http.Request, pluginsMatchResult []
pluginsProxy.ServeHTTP(w, r) pluginsProxy.ServeHTTP(w, r)
return true return true
} }
var referrerPathReg = regexp.MustCompile("^(https?)://(.+):([0-9]+)?/(.*)$")
// ServePluginWithReferrerOK check if redirectReferrer is set for the referred a plugin, if yes,
// sends a redirect to actual url. This function is useful for plugins to refer to absolute paths when
// the referrer in http.Request is set
func ServePluginWithReferrerOK(w http.ResponseWriter, r *http.Request, path string) (ok bool) {
referrer := r.Referer()
referrerPathMatch := referrerPathReg.FindStringSubmatch(referrer)
if referrerPathMatch != nil {
referrerPluginMatch := PluginsMatch.FindStringSubmatch(referrerPathMatch[4])
pluginKey := fmt.Sprintf("%s/%s", referrerPluginMatch[1], referrerPluginMatch[2])
currentPlugin, err := loadedPlugins.GetPluginByName(pluginKey)
if err != nil {
return false
}
if referrerPluginMatch != nil && currentPlugin.Rclone.RedirectReferrer {
path = fmt.Sprintf("/plugins/%s/%s/%s", referrerPluginMatch[1], referrerPluginMatch[2], path)
http.Redirect(w, r, path, http.StatusMovedPermanently)
//s.pluginsHandler.ServeHTTP(w, r)
return true
}
}
return false
}

310
fs/rc/webgui/rc.go Normal file
View file

@ -0,0 +1,310 @@
package webgui
import (
"context"
"fmt"
"os"
"path/filepath"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/rc"
)
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/listTestPlugins",
AuthRequired: true,
Fn: rcListTestPlugins,
Title: "Show currently loaded test plugins",
Help: `allows listing of test plugins with the rclone.test set to true in package.json of the plugin
This takes no parameters and returns
- loadedTestPlugins: list of currently available test plugins
Eg
rclone rc pluginsctl/listTestPlugins
`,
})
}
func rcListTestPlugins(_ context.Context, _ rc.Params) (out rc.Params, err error) {
return rc.Params{
"loadedTestPlugins": filterPlugins(loadedPlugins, func(json *PackageJSON) bool { return json.isTesting() }),
}, nil
}
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/removeTestPlugin",
AuthRequired: true,
Fn: rcRemoveTestPlugin,
Title: "Remove a test plugin",
Help: `This allows you to remove a plugin using it's name
This takes the following parameters
- name: name of the plugin in the format <author>/<plugin_name>
Eg
rclone rc pluginsctl/removeTestPlugin name=rclone/rclone-webui-react
`,
})
}
func rcRemoveTestPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
name, err := in.GetString("name")
if err != nil {
return nil, err
}
err = loadedPlugins.removePlugin(name)
if err != nil {
return nil, err
}
return nil, nil
}
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/addPlugin",
AuthRequired: true,
Fn: rcAddPlugin,
Title: "Add a plugin using url",
Help: `used for adding a plugin to the webgui
This takes the following parameters
- url: http url of the github repo where the plugin is hosted (http://github.com/rclone/rclone-webui-react)
Eg
rclone rc pluginsctl/addPlugin
`,
})
}
func rcAddPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
pluginURL, err := in.GetString("url")
if err != nil {
return nil, err
}
author, repoName, repoBranch, err := getAuthorRepoBranchGithub(pluginURL)
if err != nil {
return nil, err
}
branch, err := in.GetString("branch")
if err != nil || branch == "" {
branch = repoBranch
}
version, err := in.GetString("version")
if err != nil || version == "" {
version = "latest"
}
err = CreatePathIfNotExist(PluginsPath)
if err != nil {
return nil, err
}
// fetch and package.json
// https://raw.githubusercontent.com/rclone/rclone-webui-react/master/package.json
pluginID := fmt.Sprintf("%s/%s", author, repoName)
currentPluginPath := filepath.Join(PluginsPath, pluginID)
err = CreatePathIfNotExist(currentPluginPath)
if err != nil {
return nil, err
}
packageJSONUrl := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/package.json", author, repoName, branch)
packageJSONFilePath := filepath.Join(currentPluginPath, "package.json")
err = DownloadFile(packageJSONFilePath, packageJSONUrl)
if err != nil {
return nil, err
}
// register in plugins
// download release and save in plugins/<author>/repo-name/app
// https://api.github.com/repos/rclone/rclone-webui-react/releases/latest
releaseURL, tag, _, err := GetLatestReleaseURL(fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/%s", author, repoName, version))
if err != nil {
return nil, err
}
zipName := tag + ".zip"
zipPath := filepath.Join(currentPluginPath, zipName)
err = DownloadFile(zipPath, releaseURL)
if err != nil {
return nil, err
}
extractPath := filepath.Join(currentPluginPath, "app")
err = CreatePathIfNotExist(extractPath)
if err != nil {
return nil, err
}
err = os.RemoveAll(extractPath)
if err != nil {
fs.Logf(nil, "No previous downloads to remove")
}
fs.Logf(nil, "Unzipping plugin binary")
err = Unzip(zipPath, extractPath)
if err != nil {
return nil, err
}
err = loadedPlugins.addPlugin(pluginID, packageJSONFilePath)
if err != nil {
return nil, err
}
return nil, nil
}
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/listPlugins",
AuthRequired: true,
Fn: rcGetPlugins,
Title: "Get the list of currently loaded plugins",
Help: `This allows you to get the currently enabled plugins and their details.
This takes no parameters and returns
- loadedPlugins: list of current production plugins
- testPlugins: list of temporarily loaded development plugins, usually running on a different server.
Eg
rclone rc pluginsctl/listPlugins
`,
})
}
func rcGetPlugins(_ context.Context, _ rc.Params) (out rc.Params, err error) {
err = loadedPlugins.readFromFile()
if err != nil {
return nil, err
}
return rc.Params{
"loadedPlugins": filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { return !packageJSON.isTesting() }),
"loadedTestPlugins": filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { return packageJSON.isTesting() }),
}, nil
}
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/removePlugin",
AuthRequired: true,
Fn: rcRemovePlugin,
Title: "Remove a loaded plugin",
Help: `This allows you to remove a plugin using it's name
This takes parameters
- name: name of the plugin in the format <author>/<plugin_name>
Eg
rclone rc pluginsctl/removePlugin name=rclone/video-plugin
`,
})
}
func rcRemovePlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
name, err := in.GetString("name")
if err != nil {
return nil, err
}
err = loadedPlugins.removePlugin(name)
if err != nil {
return nil, err
}
return nil, nil
}
func init() {
rc.Add(rc.Call{
Path: "pluginsctl/getPluginsForType",
AuthRequired: true,
Fn: rcGetPluginsForType,
Title: "Get plugins with type criteria",
Help: `This shows all possible plugins by a mime type
This takes the following parameters
- type: supported mime type by a loaded plugin eg (video/mp4, audio/mp3)
- pluginType: filter plugins based on their type eg (DASHBOARD, FILE_HANDLER, TERMINAL)
and returns
- loadedPlugins: list of current production plugins
- testPlugins: list of temporarily loaded development plugins, usually running on a different server.
Eg
rclone rc pluginsctl/getPluginsForType type=video/mp4
`,
})
}
func rcGetPluginsForType(_ context.Context, in rc.Params) (out rc.Params, err error) {
handlesType, err := in.GetString("type")
if err != nil {
handlesType = ""
}
pluginType, err := in.GetString("pluginType")
if err != nil {
pluginType = ""
}
var loadedPluginsResult map[string]PackageJSON
var loadedTestPluginsResult map[string]PackageJSON
if pluginType == "" || pluginType == "FileHandler" {
loadedPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool {
for i := range packageJSON.Rclone.HandlesType {
if packageJSON.Rclone.HandlesType[i] == handlesType && !packageJSON.Rclone.Test {
return true
}
}
return false
})
loadedTestPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool {
for i := range packageJSON.Rclone.HandlesType {
if packageJSON.Rclone.HandlesType[i] == handlesType && packageJSON.Rclone.Test {
return true
}
}
return false
})
} else {
loadedPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool {
return packageJSON.Rclone.PluginType == pluginType && !packageJSON.isTesting()
})
loadedTestPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool {
return packageJSON.Rclone.PluginType == pluginType && packageJSON.isTesting()
})
}
return rc.Params{
"loadedPlugins": loadedPluginsResult,
"loadedTestPlugins": loadedTestPluginsResult,
}, nil
}

148
fs/rc/webgui/rc_test.go Normal file
View file

@ -0,0 +1,148 @@
package webgui
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/rclone/rclone/fs/rc"
"github.com/stretchr/testify/assert"
)
const testPluginName = "rclone-webui-react"
const testPluginAuthor = "rclone"
const testPluginKey = testPluginAuthor + "/" + testPluginName
const testPluginURL = "https://github.com/" + testPluginAuthor + "/" + testPluginName + "/"
func setCacheDir(t *testing.T) string {
cacheDir, err := ioutil.TempDir("", "rclone-cache-dir")
assert.Nil(t, err)
PluginsPath = filepath.Join(cacheDir, "plugins")
pluginsConfigPath = filepath.Join(cacheDir, "config")
loadedPlugins = newPlugins(availablePluginsJSONPath)
err = loadedPlugins.readFromFile()
assert.Nil(t, err)
return cacheDir
}
func cleanCacheDir(t *testing.T, cacheDir string) {
_ = os.RemoveAll(cacheDir)
}
func addPlugin(t *testing.T) {
addPlugin := rc.Calls.Get("pluginsctl/addPlugin")
assert.NotNil(t, addPlugin)
in := rc.Params{
"url": testPluginURL,
}
out, err := addPlugin.Fn(context.Background(), in)
assert.Nil(t, err)
assert.Nil(t, out)
}
func removePlugin(t *testing.T) {
addPlugin := rc.Calls.Get("pluginsctl/removePlugin")
assert.NotNil(t, addPlugin)
in := rc.Params{
"name": testPluginKey,
}
out, err := addPlugin.Fn(context.Background(), in)
assert.NotNil(t, err)
assert.Nil(t, out)
}
//func TestListTestPlugins(t *testing.T) {
// addPlugin := rc.Calls.Get("pluginsctl/listTestPlugins")
// assert.NotNil(t, addPlugin)
// in := rc.Params{}
// out, err := addPlugin.Fn(context.Background(), in)
// assert.Nil(t, err)
// expected := rc.Params{
// "loadedTestPlugins": map[string]PackageJSON{},
// }
// assert.Equal(t, expected, out)
//}
//func TestRemoveTestPlugin(t *testing.T) {
// addPlugin := rc.Calls.Get("pluginsctl/removeTestPlugin")
// assert.NotNil(t, addPlugin)
// in := rc.Params{
// "name": "",
// }
// out, err := addPlugin.Fn(context.Background(), in)
// assert.NotNil(t, err)
// assert.Nil(t, out)
//}
func TestAddPlugin(t *testing.T) {
cacheDir := setCacheDir(t)
defer cleanCacheDir(t, cacheDir)
addPlugin(t)
_, ok := loadedPlugins.LoadedPlugins[testPluginKey]
assert.True(t, ok)
//removePlugin(t)
//_, ok = loadedPlugins.LoadedPlugins[testPluginKey]
//assert.False(t, ok)
}
func TestListPlugins(t *testing.T) {
cacheDir := setCacheDir(t)
defer cleanCacheDir(t, cacheDir)
addPlugin := rc.Calls.Get("pluginsctl/listPlugins")
assert.NotNil(t, addPlugin)
in := rc.Params{}
out, err := addPlugin.Fn(context.Background(), in)
assert.Nil(t, err)
expected := rc.Params{
"loadedPlugins": map[string]PackageJSON{},
"loadedTestPlugins": map[string]PackageJSON{},
}
assert.Equal(t, expected, out)
}
func TestRemovePlugin(t *testing.T) {
cacheDir := setCacheDir(t)
defer cleanCacheDir(t, cacheDir)
addPlugin(t)
removePluginCall := rc.Calls.Get("pluginsctl/removePlugin")
assert.NotNil(t, removePlugin)
in := rc.Params{
"name": testPluginKey,
}
out, err := removePluginCall.Fn(context.Background(), in)
assert.Nil(t, err)
assert.Nil(t, out)
removePlugin(t)
assert.Equal(t, len(loadedPlugins.LoadedPlugins), 0)
}
func TestPluginsForType(t *testing.T) {
addPlugin := rc.Calls.Get("pluginsctl/getPluginsForType")
assert.NotNil(t, addPlugin)
in := rc.Params{
"type": "",
"pluginType": "FileHandler",
}
out, err := addPlugin.Fn(context.Background(), in)
assert.Nil(t, err)
assert.NotNil(t, out)
in = rc.Params{
"type": "video/mp4",
"pluginType": "",
}
_, err = addPlugin.Fn(context.Background(), in)
assert.Nil(t, err)
assert.NotNil(t, out)
}

7
fs/rc/webgui/webgui.go
View file

@ -27,9 +27,12 @@ func GetLatestReleaseURL(fetchURL string) (string, string, int, error) {
} }
results := gitHubRequest{} results := gitHubRequest{}
if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { if err := json.NewDecoder(resp.Body).Decode(&results); err != nil {
return "", "", 0, errors.New("Could not decode results from http request") return "", "", 0, errors.New("could not decode results from http request")
}
if len(results.Assets) < 1 {
return "", "", 0, errors.New("could not find an asset in the release. " +
"check if asset was successfully added in github release assets")
} }
res := results.Assets[0].BrowserDownloadURL res := results.Assets[0].BrowserDownloadURL
tag := results.TagName tag := results.TagName
size := results.Assets[0].Size size := results.Assets[0].Size