TrueCloudLab/frostfs-http-gw
17
1
Fork 18
Code Issues 11 Pull requests 6 Projects Releases 16 Packages 1 Activity Actions

[#137] Add index page support
All checks were successful
/ DCO (pull_request) Successful in 1m6s
Details
/ Builds (pull_request) Successful in 58s
Details
/ Vulncheck (pull_request) Successful in 1m29s
Details
/ Lint (pull_request) Successful in 2m15s
Details
/ Tests (pull_request) Successful in 1m33s
Details

Browse source 
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2024-09-26 17:32:27 +03:00
parent 843708a558
commit 329f43c079
18 changed files with 739 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

169
internal/handler/browse.go Normal file
View file

@ -0,0 +1,169 @@
package handler
import (
"cmp"
_ "embed"
"html/template"
"net/url"
"strconv"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"github.com/docker/go-units"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
"golang.org/x/exp/slices"
)
const (
dateFormat = "02-01-2006 15:04"
attrOID, attrCreated, attrFileName, attrSize = "OID", "Created", "FileName", "Size"
)
type (
BrowsePageData struct {
BucketName,
Prefix string
Objects []ResponseObject
}
ResponseObject struct {
OID string
Created string
FileName string
Size string
}
)
//go:embed templates/index.gotmpl
var defaultTemplate string
func parseTimestamp(tstamp string) (time.Time, error) {
millis, err := strconv.ParseInt(tstamp, 10, 64)
if err != nil {
return time.Time{}, err
}
return time.UnixMilli(millis), nil
}
func NewResponseObject(nodes map[string]string) ResponseObject {
return ResponseObject{
OID: nodes[attrOID],
Created: nodes[attrCreated],
FileName: nodes[attrFileName],
Size: nodes[attrSize],
}
}
func formatTimestamp(strdate string) string {
date, err := parseTimestamp(strdate)
if err != nil || date.IsZero() {
return ""
}
return date.Format(dateFormat)
}
func formatSize(strsize string) string {
size, err := strconv.ParseFloat(strsize, 64)
if err != nil {
return ""
}
return units.HumanSize(size)
}
func parentDir(prefix string) string {
index := strings.LastIndex(prefix, "/")
if index == -1 {
return ""
}
return prefix[index:]
}
func trimPrefix(encPrefix string) string {
prefix, err := url.PathUnescape(encPrefix)
if err != nil {
return ""
}
slashIndex := strings.LastIndex(encPrefix, "/")
if slashIndex == -1 {
return ""
}
return prefix[:slashIndex]
}
func urlencode(prefix, filename, size, created string) string {
res := prefix
if prefix != "" {
res += "/"
}
res += filename
if size == "" && created == "" {
res += "/"
}
return url.PathEscape(res)
}
func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) {
var log = h.log.With(zap.String("bucket", bucketInfo.Name))
ctx := utils.GetContextFromRequest(c)
nodes, err := h.listObjects(ctx, bucketInfo, prefix)
if err != nil {
logAndSendBucketError(c, log, err)
return
}
respObjects := make([]ResponseObject, len(nodes))
for i, node := range nodes {
respObjects[i] = NewResponseObject(node)
}
slices.SortFunc(respObjects, func(a, b ResponseObject) int {
aIsDir := a.Size == ""
bIsDir := b.Size == ""
// return root object first
if a.FileName == "" {
return -1
} else if b.FileName == "" {
return 1
}
// dirs objects go first
if aIsDir && !bIsDir {
return -1
} else if !aIsDir && bIsDir {
return 1
}
return cmp.Compare(a.FileName, b.FileName)
})
indexTemplate := h.config.IndexPageTemplate()
if indexTemplate == "" {
indexTemplate = defaultTemplate
}
tmpl, err := template.New("index").Funcs(template.FuncMap{
"formatTimestamp": formatTimestamp,
"formatSize": formatSize,
"trimPrefix": trimPrefix,
"urlencode": urlencode,
"parentDir": parentDir,
}).Parse(indexTemplate)
if err != nil {
logAndSendBucketError(c, log, err)
return
}
if err = tmpl.Execute(c, &BrowsePageData{
BucketName: bucketInfo.Name,
Prefix: prefix,
Objects: respObjects,
}); err != nil {
logAndSendBucketError(c, log, err)
return
}
}

57
internal/handler/handler.go
View file

@ -30,6 +30,9 @@ type Config interface {
DefaultTimestamp() bool
ZipCompression() bool
ClientCut() bool
IndexPageEnabled() bool
IndexPageTemplatePath() string
IndexPageTemplate() string
BufferMaxSizeForPut() uint64
NamespaceHeader() string
}
@ -208,41 +211,47 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ
// byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that
// prepares request and object address to it.
func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
var (
bucketname = req.UserValue("cid").(string)
key = req.UserValue("oid").(string)
bucketname = c.UserValue("cid").(string)
key = c.UserValue("oid").(string)
log = h.log.With(zap.String("bucketname", bucketname), zap.String("key", key))
download = c.QueryArgs().GetBool("download")
)
unescapedKey, err := url.QueryUnescape(key)
if err != nil {
logAndSendBucketError(req, log, err)
logAndSendBucketError(c, log, err)
return
}
ctx := utils.GetContextFromRequest(req)
ctx := utils.GetContextFromRequest(c)
bktInfo, err := h.getBucketInfo(ctx, bucketname, log)
if err != nil {
logAndSendBucketError(req, log, err)
logAndSendBucketError(c, log, err)
return
}
if h.config.IndexPageEnabled() && !download && string(c.Method()) != fasthttp.MethodHead {
if isDir(unescapedKey) || isContainerRoot(unescapedKey) {
h.browseObjects(c, bktInfo, unescapedKey)
return
}
}
foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey)
if err != nil {
if errors.Is(err, tree.ErrNodeAccessDenied) {
response.Error(req, "Access Denied", fasthttp.StatusForbidden)
return
response.Error(c, "Access Denied", fasthttp.StatusForbidden)
} else {
log.Error(logs.GetLatestObjectVersion, zap.Error(err))
response.Error(c, "object wasn't found", fasthttp.StatusNotFound)
}
log.Error(logs.GetLatestObjectVersion, zap.Error(err))
response.Error(req, "object wasn't found", fasthttp.StatusNotFound)
return
}
if foundOid.DeleteMarker {
log.Error(logs.ObjectWasDeleted)
response.Error(req, "object deleted", fasthttp.StatusNotFound)
response.Error(c, "object deleted", fasthttp.StatusNotFound)
return
}
@ -250,7 +259,7 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context,
addr.SetContainer(bktInfo.CID)
addr.SetObject(foundOid.OID)
f(ctx, *h.newRequest(req, log), addr)
f(ctx, *h.newRequest(c, log), addr)
}
// byAttribute is a wrapper similar to byAddress.
@ -379,3 +388,25 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket
return bktInfo, err
}
func (h *Handler) listObjects(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]map[string]string, error) {
nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
if err != nil {
return nil, err
}
var objects = make([]map[string]string, 0, len(nodes))
for _, node := range nodes {
meta := node.GetMeta()
if meta == nil {
continue
}
var obj = make(map[string]string, len(meta))
for _, m := range meta {
obj[m.GetKey()] = string(m.GetValue())
}
objects = append(objects, obj)
}
return objects, nil
}

16
internal/handler/handler_test.go
View file

@ -12,6 +12,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
@ -37,6 +38,10 @@ func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree
return nil, nil
}
func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) {
return nil, nil
}
type configMock struct {
}
@ -48,6 +53,17 @@ func (c *configMock) ZipCompression() bool {
return false
}
func (c *configMock) IndexPageEnabled() bool {
return false
}
func (c *configMock) IndexPageTemplatePath() string {
return ""
}
func (c *configMock) IndexPageTemplate() string {
return ""
}
func (c *configMock) ClientCut() bool {
return false
}

90
internal/handler/templates/index.gotmpl Normal file
View file

@ -0,0 +1,90 @@
{{$bucketName := .BucketName}}
{{ $prefix := trimPrefix .Prefix }}
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8"/>
<title>Index of s3://{{$bucketName}}{{if $prefix}}/{{$prefix}}/{{end}}</title>
<style>
table {
width: 80%;
border-collapse: collapse;
}
body {
background: #f2f2f2;
}
table, th, td {
border: 0 solid transparent;
}
th, td {
padding: 10px;
text-align: left;
}
th {
background-color: #c3bcbc;
}
tr:nth-child(even) {background-color: #ebe7e7;}
</style>
</head>
<body>
<h1>Index of s3://{{$bucketName}}/{{if $prefix}}{{$prefix}}/{{end}}</h1>
<table>
<thead>
<tr>
<th>Filename</th>
<th>Size</th>
<th>Created</th>
<th>Download</th>
</tr>
</thead>
<tbody>
{{ $trimmedPrefix := trimPrefix $prefix }}
{{if $trimmedPrefix }}
<tr>
<td><a href="/get/{{$bucketName}}/{{ $trimmedPrefix }}/">../</a></td>
<td></td>
<td></td>
</tr>
{{else}}
<tr>
<td><a href="/get/{{ $bucketName}}/">../</a></td>
<td></td>
<td></td>
</tr>
{{end}}
{{range .Objects}}
<tr>
<td>
{{if and (not .FileName) (not .Size)}}
{{if not .Created}}
<a href="/get/{{ $bucketName}}/{{ urlencode $prefix .FileName .Size .Created }}">
/
</a>
{{else}}
{{/* current dir info */}}
.
{{end}}
{{else if and (not .FileName) .Size}}
{{/* this is an object with body of current dir */}}
..{{parentDir $prefix}}/
{{else}}
<a href="/get/{{ $bucketName}}/{{ urlencode $prefix .FileName .Size .Created}}">
{{.FileName}}{{if and (not .Size) (not .Created)}}/{{end}}
</a>
{{end}}
</td>
<td>{{ formatSize .Size }}</td>
<td>{{ formatTimestamp .Created }}</td>
<td>
{{ if .Size }}
<a href="/get/{{ $bucketName}}/{{ urlencode $prefix .FileName .Size }}?download=true">
Link
</a>
{{ end }}
</td>
</tr>
{{end}}
</tbody>
</table>
</body>
</html>

34
internal/handler/upload.go
View file

@ -43,22 +43,22 @@ func (pr *putResponse) encode(w io.Writer) error {
}
// Upload handles multipart upload request.
func (h *Handler) Upload(req *fasthttp.RequestCtx) {
func (h *Handler) Upload(c *fasthttp.RequestCtx) {
var (
file MultipartFile
idObj oid.ID
addr oid.Address
scid, _ = req.UserValue("cid").(string)
scid, _ = c.UserValue("cid").(string)
log = h.log.With(zap.String("cid", scid))
bodyStream = req.RequestBodyStream()
bodyStream = c.RequestBodyStream()
drainBuf = make([]byte, drainBufSize)
)
ctx := utils.GetContextFromRequest(req)
ctx := utils.GetContextFromRequest(c)
bktInfo, err := h.getBucketInfo(ctx, scid, log)
if err != nil {
logAndSendBucketError(req, log, err)
logAndSendBucketError(c, log, err)
return
}
@ -75,21 +75,21 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) {
zap.Error(err),
)
}()
boundary := string(req.Request.Header.MultipartFormBoundary())
boundary := string(c.Request.Header.MultipartFormBoundary())
if file, err = fetchMultipartFile(h.log, bodyStream, boundary); err != nil {
log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err))
response.Error(req, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest)
response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest)
return
}
filtered, err := filterHeaders(h.log, &req.Request.Header)
filtered, err := filterHeaders(h.log, &c.Request.Header)
if err != nil {
log.Error(logs.CouldNotProcessHeaders, zap.Error(err))
response.Error(req, err.Error(), fasthttp.StatusBadRequest)
response.Error(c, err.Error(), fasthttp.StatusBadRequest)
return
}
now := time.Now()
if rawHeader := req.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil {
if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil {
if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil {
log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err))
} else {
@ -97,9 +97,9 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) {
}
}
if err = utils.PrepareExpirationHeader(req, h.frostfs, filtered, now); err != nil {
if err = utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil {
log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err))
response.Error(req, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest)
response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest)
return
}
@ -143,7 +143,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) {
}
if idObj, err = h.frostfs.CreateObject(ctx, prm); err != nil {
h.handlePutFrostFSErr(req, err)
h.handlePutFrostFSErr(c, err)
return
}
@ -151,9 +151,9 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) {
addr.SetContainer(bktInfo.CID)
// Try to return the response, otherwise, if something went wrong, throw an error.
if err = newPutResponse(addr).encode(req); err != nil {
if err = newPutResponse(addr).encode(c); err != nil {
log.Error(logs.CouldNotEncodeResponse, zap.Error(err))
response.Error(req, "could not encode response", fasthttp.StatusBadRequest)
response.Error(c, "could not encode response", fasthttp.StatusBadRequest)
return
}
@ -170,8 +170,8 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) {
}
}
// Report status code and content type.
req.Response.SetStatusCode(fasthttp.StatusOK)
req.Response.Header.SetContentType(jsonHeader)
c.Response.SetStatusCode(fasthttp.StatusOK)
c.Response.Header.SetContentType(jsonHeader)
}
func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) {

7
internal/handler/utils.go
View file

@ -38,6 +38,13 @@ func bearerToken(ctx context.Context) *bearer.Token {
return nil
}
func isDir(name string) bool {
return strings.HasSuffix(name, "/")
}
func isContainerRoot(key string) bool {
return key == ""
}
func isValidToken(s string) bool {
for _, c := range s {
if c <= ' ' || c > 127 {