rclone/cmd/serve/s3/server.go

// Package s3 implements a fake s3 server for rclone
package s3

import (
	"context"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"math/rand"
	"net/http"
	"strings"

	"github.com/go-chi/chi/v5"
	"github.com/rclone/gofakes3"
	"github.com/rclone/gofakes3/signature"
	"github.com/rclone/rclone/cmd/serve/proxy"
	"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/hash"
	httplib "github.com/rclone/rclone/lib/http"
	"github.com/rclone/rclone/vfs"
	"github.com/rclone/rclone/vfs/vfscommon"
)

type ctxKey int

const (
	ctxKeyID ctxKey = iota
)

// Options contains options for the http Server
type Options struct {
	//TODO add more options
	pathBucketMode bool
	hashName       string
	hashType       hash.Type
	authPair       []string
	noCleanup      bool
	Auth           httplib.AuthConfig
	HTTP           httplib.Config
}

// Server is a s3.FileSystem interface
type Server struct {
	server   *httplib.Server
	f        fs.Fs
	_vfs     *vfs.VFS // don't use directly, use getVFS
	faker    *gofakes3.GoFakeS3
	handler  http.Handler
	proxy    *proxy.Proxy
	ctx      context.Context // for global config
	s3Secret string
}

// Make a new S3 Server to serve the remote
func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error) {
	w := &Server{
		f:   f,
		ctx: ctx,
	}

	if len(opt.authPair) == 0 {
		fs.Logf("serve s3", "No auth provided so allowing anonymous access")
	} else {
		w.s3Secret = getAuthSecret(opt.authPair)
	}

	var newLogger logger
	w.faker = gofakes3.New(
		newBackend(w, opt),
		gofakes3.WithHostBucket(!opt.pathBucketMode),
		gofakes3.WithLogger(newLogger),
		gofakes3.WithRequestID(rand.Uint64()),
		gofakes3.WithoutVersioning(),
		gofakes3.WithV4Auth(authlistResolver(opt.authPair)),
		gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied
	)

	w.handler = http.NewServeMux()
	w.handler = w.faker.Server()

	if proxyflags.Opt.AuthProxy != "" {
		w.proxy = proxy.New(ctx, &proxyflags.Opt)
		// proxy auth middleware
		w.handler = proxyAuthMiddleware(w.handler, w)
		w.handler = authPairMiddleware(w.handler, w)
	} else {
		w._vfs = vfs.New(f, &vfscommon.Opt)

		if len(opt.authPair) > 0 {
			w.faker.AddAuthKeys(authlistResolver(opt.authPair))
		}
	}

	w.server, err = httplib.NewServer(ctx,
		httplib.WithConfig(opt.HTTP),
		httplib.WithAuth(opt.Auth),
	)
	if err != nil {
		return nil, fmt.Errorf("failed to init server: %w", err)
	}

	return w, nil
}

func (w *Server) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) {
	if w._vfs != nil {
		return w._vfs, nil
	}

	value := ctx.Value(ctxKeyID)
	if value == nil {
		return nil, errors.New("no VFS found in context")
	}

	VFS, ok := value.(*vfs.VFS)
	if !ok {
		return nil, fmt.Errorf("context value is not VFS: %#v", value)
	}
	return VFS, nil
}

// auth does proxy authorization
func (w *Server) auth(accessKeyID string) (value interface{}, err error) {
	VFS, _, err := w.proxy.Call(stringToMd5Hash(accessKeyID), accessKeyID, false)
	if err != nil {
		return nil, err
	}
	return VFS, err
}

// Bind register the handler to http.Router
func (w *Server) Bind(router chi.Router) {
	router.Handle("/*", w.handler)
}

// Serve serves the s3 server
func (w *Server) Serve() error {
	w.server.Serve()
	fs.Logf(w.f, "Starting s3 server on %s", w.server.URLs())
	return nil
}

func authPairMiddleware(next http.Handler, ws *Server) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		accessKey, _ := parseAccessKeyID(r)
		// set the auth pair
		authPair := map[string]string{
			accessKey: ws.s3Secret,
		}
		ws.faker.AddAuthKeys(authPair)
		next.ServeHTTP(w, r)
	})
}

func proxyAuthMiddleware(next http.Handler, ws *Server) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		accessKey, _ := parseAccessKeyID(r)
		value, err := ws.auth(accessKey)
		if err != nil {
			fs.Infof(r.URL.Path, "%s: Auth failed: %v", r.RemoteAddr, err)
		}
		if value != nil {
			r = r.WithContext(context.WithValue(r.Context(), ctxKeyID, value))
		}

		next.ServeHTTP(w, r)
	})
}

func parseAccessKeyID(r *http.Request) (accessKey string, error signature.ErrorCode) {
	v4Auth := r.Header.Get("Authorization")
	req, err := signature.ParseSignV4(v4Auth)
	if err != signature.ErrNone {
		return "", err
	}

	return req.Credential.GetAccessKey(), signature.ErrNone
}

func stringToMd5Hash(s string) string {
	hasher := md5.New()
	hasher.Write([]byte(s))
	return hex.EncodeToString(hasher.Sum(nil))
}

func getAuthSecret(authPair []string) string {
	if len(authPair) == 0 {
		return ""
	}

	splited := strings.Split(authPair[0], ",")
	if len(splited) != 2 {
		return ""
	}

	secret := strings.TrimSpace(splited[1])
	return secret
}