alexvanin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

serve http command to serve a remote over HTTP

Browse source 
This implements a basic webserver to serve an rclone remote over HTTP.

It also sets up the framework for adding more types of server later.
This commit is contained in:
Nick Craig-Wood 2016-12-06 18:00:24 +00:00
parent a06bcd4c57
commit 6669165b6b
22 changed files with 494 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmd/all/all.go
View file

@ -38,6 +38,7 @@ import (
_ "github.com/ncw/rclone/cmd/rcat"
_ "github.com/ncw/rclone/cmd/rmdir"
_ "github.com/ncw/rclone/cmd/rmdirs"
_ "github.com/ncw/rclone/cmd/serve"
_ "github.com/ncw/rclone/cmd/sha1sum"
_ "github.com/ncw/rclone/cmd/size"
_ "github.com/ncw/rclone/cmd/sync"

251
cmd/serve/http/http.go Normal file
View file

@ -0,0 +1,251 @@
package http
import (
"fmt"
"html/template"
"io"
"log"
"net/http"
"path"
"strconv"
"strings"
"time"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
// Globals
var (
bindAddress = "localhost:8080"
readWrite = false
)
func init() {
Command.Flags().StringVarP(&bindAddress, "addr", "", bindAddress, "IPaddress:Port to bind server to.")
// Command.Flags().BoolVarP(&readWrite, "rw", "", readWrite, "Serve in read/write mode.")
}
// Command definition for cobra
var Command = &cobra.Command{
Use: "http remote:path",
Short: `Serve the remote over HTTP.`,
Long: `rclone serve http implements a basic web server to serve the remote
over HTTP. This can be viewed in a web browser or you can make a
remote of type http read from it.
Use --addr to specify which IP address and port the server should
listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all
IPs. By default it only listens on localhost.
You can use the filter flags (eg --include, --exclude) to control what
is served.
The server will log errors. Use -v to see access logs.
--bwlimit will be respected for file transfers. Use --stats to
control the stats printing.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
f := cmd.NewFsSrc(args)
cmd.Run(false, true, command, func() error {
s := server{
f: f,
bindAddress: bindAddress,
readWrite: readWrite,
}
s.serve()
return nil
})
},
}
// server contains everything to run the server
type server struct {
f fs.Fs
bindAddress string
readWrite bool
}
// serve creates the http server
func (s *server) serve() {
mux := http.NewServeMux()
mux.HandleFunc("/", s.handler)
// FIXME make a transport?
httpServer := &http.Server{
Addr: s.bindAddress,
Handler: mux,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
}
fs.Logf(s.f, "Serving on http://%s/", bindAddress)
log.Fatal(httpServer.ListenAndServe())
}
// handler reads incoming requests and dispatches them
func (s *server) handler(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" && r.Method != "HEAD" {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
urlPath := r.URL.Path
isDir := strings.HasSuffix(urlPath, "/")
remote := strings.Trim(urlPath, "/")
if isDir {
s.serveDir(w, r, remote)
} else {
s.serveFile(w, r, remote)
}
}
// entry is a directory entry
type entry struct {
remote string
URL string
Leaf string
}
// entries represents a directory
type entries []entry
// indexPage is a directory listing template
var indexPage = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{{ .Title }}</title>
</head>
<body>
<h1>{{ .Title }}</h1>
{{ range $i := .Entries }}<a href="{{ $i.URL }}">{{ $i.Leaf }}</a><br />
{{ end }}</body>
</html>
`
// indexTemplate is the instantiated indexPage
var indexTemplate = template.Must(template.New("index").Parse(indexPage))
// indexData is used to fill in the indexTemplate
type indexData struct {
Title string
Entries entries
}
// error returns an http.StatusInternalServerError and logs the error
func internalError(what interface{}, w http.ResponseWriter, text string, err error) {
fs.Stats.Error()
fs.Errorf(what, "%s: %v", text, err)
http.Error(w, text+".", http.StatusInternalServerError)
}
// serveDir serves a directory index at dirRemote
func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
// Check the directory is included in the filters
if !fs.Config.Filter.IncludeDirectory(dirRemote) {
fs.Infof(dirRemote, "%s: Directory not found (filtered)", r.RemoteAddr)
http.Error(w, "Directory not found", http.StatusNotFound)
return
}
// List the directory
dirEntries, err := fs.ListDirSorted(s.f, false, dirRemote)
if err == fs.ErrorDirNotFound {
fs.Infof(dirRemote, "%s: Directory not found", r.RemoteAddr)
http.Error(w, "Directory not found", http.StatusNotFound)
return
} else if err != nil {
internalError(dirRemote, w, "Failed to list directory", err)
return
}
var out entries
for _, o := range dirEntries {
remote := strings.Trim(o.Remote(), "/")
leaf := path.Base(remote)
urlRemote := leaf
if _, ok := o.(*fs.Dir); ok {
leaf += "/"
urlRemote += "/"
}
out = append(out, entry{remote: remote, URL: urlRemote, Leaf: leaf})
}
// Account the transfer
fs.Stats.Transferring(dirRemote)
defer fs.Stats.DoneTransferring(dirRemote, true)
fs.Infof(dirRemote, "%s: Serving directory", r.RemoteAddr)
err = indexTemplate.Execute(w, indexData{
Entries: out,
Title: fmt.Sprintf("Directory listing of /%s", dirRemote),
})
if err != nil {
internalError(dirRemote, w, "Failed to render template", err)
return
}
}
// serveFile serves a file object at remote
func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
// FIXME could cache the directories and objects...
obj, err := s.f.NewObject(remote)
if err == fs.ErrorObjectNotFound {
fs.Infof(remote, "%s: File not found", r.RemoteAddr)
http.Error(w, "File not found", http.StatusNotFound)
return
} else if err != nil {
internalError(remote, w, "Failed to find file", err)
return
}
// Check the object is included in the filters
if !fs.Config.Filter.IncludeObject(obj) {
fs.Infof(remote, "%s: File not found (filtered)", r.RemoteAddr)
http.Error(w, "File not found", http.StatusNotFound)
return
}
// Set content length since we know how long the object is
w.Header().Set("Content-Length", strconv.FormatInt(obj.Size(), 10))
// Set content type
mimeType := fs.MimeType(obj)
if mimeType == "application/octet-stream" && path.Ext(remote) == "" {
// Leave header blank so http server guesses
} else {
w.Header().Set("Content-Type", mimeType)
}
// If HEAD no need to read the object since we have set the headers
if r.Method == "HEAD" {
return
}
// open the object
in, err := obj.Open()
if err != nil {
internalError(remote, w, "Failed to open file", err)
return
}
defer func() {
err := in.Close()
if err != nil {
fs.Errorf(remote, "Failed to close file: %v", err)
}
}()
// Account the transfer
fs.Stats.Transferring(remote)
defer fs.Stats.DoneTransferring(remote, true)
in = fs.NewAccount(in, obj).WithBuffer() // account the transfer
// Copy the contents of the object to the output
fs.Infof(remote, "%s: Serving file", r.RemoteAddr)
_, err = io.Copy(w, in)
if err != nil {
fs.Errorf(remote, "Failed to write file: %v", err)
}
}

170
cmd/serve/http/http_test.go Normal file
View file

@ -0,0 +1,170 @@
package http
import (
"flag"
"io/ioutil"
"net"
"net/http"
"strings"
"testing"
"time"
"github.com/ncw/rclone/fs"
_ "github.com/ncw/rclone/local"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var updateGolden = flag.Bool("updategolden", false, "update golden files for regression test")
const (
testBindAddress = "localhost:51777"
testURL = "http://" + testBindAddress + "/"
)
func startServer(t *testing.T, f fs.Fs) {
s := server{
f: f,
bindAddress: testBindAddress,
readWrite: false,
}
go s.serve()
// try to connect to the test server
pause := time.Millisecond
for i := 0; i < 10; i++ {
conn, err := net.Dial("tcp", testBindAddress)
if err == nil {
_ = conn.Close()
return
}
// t.Logf("couldn't connect, sleeping for %v: %v", pause, err)
time.Sleep(pause)
pause *= 2
}
t.Fatal("couldn't connect to server")
}
func TestInit(t *testing.T) {
// Configure the remote
fs.LoadConfig()
// fs.Config.LogLevel = fs.LogLevelDebug
// fs.Config.DumpHeaders = true
// fs.Config.DumpBodies = true
// exclude files called hidden.txt and directories called hidden
require.NoError(t, fs.Config.Filter.AddRule("- hidden.txt"))
require.NoError(t, fs.Config.Filter.AddRule("- hidden/**"))
// Create a test Fs
f, err := fs.NewFs("testdata/files")
require.NoError(t, err)
startServer(t, f)
}
// check body against the file, or re-write body if -updategolden is
// set.
func checkGolden(t *testing.T, fileName string, got []byte) {
if *updateGolden {
t.Logf("Updating golden file %q", fileName)
err := ioutil.WriteFile(fileName, got, 0666)
require.NoError(t, err)
} else {
want, err := ioutil.ReadFile(fileName)
require.NoError(t, err)
wants := strings.Split(string(want), "\n")
gots := strings.Split(string(got), "\n")
assert.Equal(t, wants, gots, fileName)
}
}
func TestGets(t *testing.T) {
for _, test := range []struct {
URL string
Status int
Golden string
Method string
}{
{
URL: "",
Status: http.StatusOK,
Golden: "testdata/golden/index.html",
},
{
URL: "notfound",
Status: http.StatusNotFound,
Golden: "testdata/golden/notfound.html",
},
{
URL: "dirnotfound/",
Status: http.StatusNotFound,
Golden: "testdata/golden/dirnotfound.html",
},
{
URL: "hidden/",
Status: http.StatusNotFound,
Golden: "testdata/golden/hiddendir.html",
},
{
URL: "one%25.txt",
Status: http.StatusOK,
Golden: "testdata/golden/one.txt",
},
{
URL: "hidden.txt",
Status: http.StatusNotFound,
Golden: "testdata/golden/hidden.txt",
},
{
URL: "three/",
Status: http.StatusOK,
Golden: "testdata/golden/three.html",
},
{
URL: "three/a.txt",
Status: http.StatusOK,
Golden: "testdata/golden/a.txt",
},
{
URL: "",
Method: "HEAD",
Status: http.StatusOK,
Golden: "testdata/golden/indexhead.txt",
},
{
URL: "one%25.txt",
Method: "HEAD",
Status: http.StatusOK,
Golden: "testdata/golden/onehead.txt",
},
{
URL: "",
Method: "POST",
Status: http.StatusMethodNotAllowed,
Golden: "testdata/golden/indexpost.txt",
},
{
URL: "one%25.txt",
Method: "POST",
Status: http.StatusMethodNotAllowed,
Golden: "testdata/golden/onepost.txt",
},
} {
method := test.Method
if method == "" {
method = "GET"
}
req, err := http.NewRequest(method, testURL+test.URL, nil)
require.NoError(t, err)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
assert.Equal(t, test.Status, resp.StatusCode, test.Golden)
body, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
checkGolden(t, test.Golden, body)
}
}

1
cmd/serve/http/testdata/files/hidden.txt vendored Normal file
View file

@ -0,0 +1 @@
hidden

1
cmd/serve/http/testdata/files/hidden/file.txt vendored Normal file
View file

@ -0,0 +1 @@
hiddenfile

1
cmd/serve/http/testdata/files/one%.txt vendored Normal file
View file

@ -0,0 +1 @@
one%

1
cmd/serve/http/testdata/files/three/a.txt vendored Normal file
View file

@ -0,0 +1 @@
three

1
cmd/serve/http/testdata/files/three/b.txt vendored Normal file
View file

@ -0,0 +1 @@
threeb

1
cmd/serve/http/testdata/files/two.txt vendored Normal file
View file

@ -0,0 +1 @@
two

1
cmd/serve/http/testdata/golden/a.txt vendored Normal file
View file

@ -0,0 +1 @@
three

1
cmd/serve/http/testdata/golden/dirnotfound.html vendored Normal file
View file

@ -0,0 +1 @@
Directory not found

1
cmd/serve/http/testdata/golden/hidden.txt vendored Normal file
View file

@ -0,0 +1 @@
File not found

1
cmd/serve/http/testdata/golden/hiddendir.html vendored Normal file
View file

@ -0,0 +1 @@
Directory not found

13
cmd/serve/http/testdata/golden/index.html vendored Normal file
View file

@ -0,0 +1,13 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Directory listing of /</title>
</head>
<body>
<h1>Directory listing of /</h1>
<a href="one%25.txt">one%.txt</a><br />
<a href="three/">three/</a><br />
<a href="two.txt">two.txt</a><br />
</body>
</html>

0
cmd/serve/http/testdata/golden/indexhead.txt vendored Normal file
View file

1
cmd/serve/http/testdata/golden/indexpost.txt vendored Normal file
View file

@ -0,0 +1 @@
Method not allowed

1
cmd/serve/http/testdata/golden/notfound.html vendored Normal file
View file

@ -0,0 +1 @@
File not found

1
cmd/serve/http/testdata/golden/one.txt vendored Normal file
View file

@ -0,0 +1 @@
one%

0
cmd/serve/http/testdata/golden/onehead.txt vendored Normal file
View file

1
cmd/serve/http/testdata/golden/onepost.txt vendored Normal file
View file

@ -0,0 +1 @@
Method not allowed

12
cmd/serve/http/testdata/golden/three.html vendored Normal file
View file

@ -0,0 +1,12 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Directory listing of /three</title>
</head>
<body>
<h1>Directory listing of /three</h1>
<a href="a.txt">a.txt</a><br />
<a href="b.txt">b.txt</a><br />
</body>
</html>

33
cmd/serve/serve.go Normal file
View file

@ -0,0 +1,33 @@
package serve
import (
"errors"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/serve/http"
"github.com/spf13/cobra"
)
func init() {
Command.AddCommand(http.Command)
cmd.Root.AddCommand(Command)
}
// Command definition for cobra
var Command = &cobra.Command{
Use: "serve <protocol> [opts] <remote>",
Short: `Serve a remote over a protocol.`,
Long: `rclone serve is used to serve a remote over a given protocol. This
command requires the use of a subcommand to specify the protocol, eg
rclone serve http remote:
Each subcommand has its own options which you can see in their help.
`,
RunE: func(command *cobra.Command, args []string) error {
if len(args) == 0 {
return errors.New("serve requires a protocol, eg 'rclone serve http remote:'")
}
return errors.New("unknown protocol")
},
}