[#151] index page: add native protocol support
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
parent
8fe8f2dcc2
commit
b188457a15
12 changed files with 420 additions and 92 deletions
|
@ -89,15 +89,16 @@ type (
|
|||
appSettings struct {
|
||||
reconnectInterval time.Duration
|
||||
|
||||
mu sync.RWMutex
|
||||
defaultTimestamp bool
|
||||
zipCompression bool
|
||||
clientCut bool
|
||||
returnIndexPage bool
|
||||
indexPageTemplate string
|
||||
bufferMaxSizeForPut uint64
|
||||
namespaceHeader string
|
||||
defaultNamespaces []string
|
||||
mu sync.RWMutex
|
||||
defaultTimestamp bool
|
||||
zipCompression bool
|
||||
clientCut bool
|
||||
returnIndexPage bool
|
||||
indexPageS3Template string
|
||||
indexPageNativeTemplate string
|
||||
bufferMaxSizeForPut uint64
|
||||
namespaceHeader string
|
||||
defaultNamespaces []string
|
||||
}
|
||||
)
|
||||
|
||||
|
@ -159,7 +160,7 @@ func newApp(ctx context.Context, opt ...Option) App {
|
|||
a.initResolver()
|
||||
a.initMetrics()
|
||||
a.initTracing(ctx)
|
||||
a.loadIndexPageTemplate()
|
||||
a.loadIndexPageTemplates()
|
||||
|
||||
return a
|
||||
}
|
||||
|
@ -188,13 +189,22 @@ func (s *appSettings) IndexPageEnabled() bool {
|
|||
return s.returnIndexPage
|
||||
}
|
||||
|
||||
func (s *appSettings) IndexPageTemplate() string {
|
||||
func (s *appSettings) IndexPageS3Template() string {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
if s.indexPageTemplate == "" {
|
||||
if s.indexPageS3Template == "" {
|
||||
return templates.DefaultIndexTemplate
|
||||
}
|
||||
return s.indexPageTemplate
|
||||
return s.indexPageS3Template
|
||||
}
|
||||
|
||||
func (s *appSettings) IndexPageNativeTemplate() string {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
if s.indexPageNativeTemplate == "" {
|
||||
return templates.DefaultIndexNativeTemplate
|
||||
}
|
||||
return s.indexPageNativeTemplate
|
||||
}
|
||||
|
||||
func (s *appSettings) setZipCompression(val bool) {
|
||||
|
@ -209,30 +219,50 @@ func (s *appSettings) setReturnIndexPage(val bool) {
|
|||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *appSettings) setIndexTemplate(val string) {
|
||||
func (s *appSettings) setIndexS3Template(val string) {
|
||||
s.mu.Lock()
|
||||
s.indexPageTemplate = val
|
||||
s.indexPageS3Template = val
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (a *app) loadIndexPageTemplate() {
|
||||
func (s *appSettings) setIndexNativeTemplate(val string) {
|
||||
s.mu.Lock()
|
||||
s.indexPageNativeTemplate = val
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (a *app) loadIndexPageTemplates() {
|
||||
if !a.settings.IndexPageEnabled() {
|
||||
return
|
||||
}
|
||||
reader, err := os.Open(a.cfg.GetString(cfgIndexPageTemplatePath))
|
||||
|
||||
pathS3 := a.cfg.GetString(cfgIndexPageS3TemplatePath)
|
||||
tmpl, err := a.readTemplate(pathS3)
|
||||
if err != nil {
|
||||
a.settings.setIndexTemplate("")
|
||||
a.settings.setIndexS3Template("")
|
||||
a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err))
|
||||
return
|
||||
} else {
|
||||
a.settings.setIndexS3Template(string(tmpl))
|
||||
a.log.Info(logs.SetCustomIndexPageTemplate, zap.String("path", pathS3))
|
||||
}
|
||||
tmpl, err := io.ReadAll(reader)
|
||||
|
||||
pathNative := a.cfg.GetString(cfgIndexPageNativeTemplatePath)
|
||||
tmpl, err = a.readTemplate(pathNative)
|
||||
if err != nil {
|
||||
a.settings.setIndexTemplate("")
|
||||
a.settings.setIndexNativeTemplate("")
|
||||
a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err))
|
||||
return
|
||||
} else {
|
||||
a.settings.setIndexNativeTemplate(string(tmpl))
|
||||
a.log.Info(logs.SetCustomIndexPageTemplate, zap.String("path", pathNative))
|
||||
}
|
||||
a.settings.setIndexTemplate(string(tmpl))
|
||||
a.log.Info(logs.SetCustomIndexPageTemplate)
|
||||
}
|
||||
|
||||
func (a *app) readTemplate(path string) ([]byte, error) {
|
||||
reader, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(reader)
|
||||
}
|
||||
|
||||
func (s *appSettings) ClientCut() bool {
|
||||
|
@ -543,7 +573,7 @@ func (a *app) configReload(ctx context.Context) {
|
|||
|
||||
a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled))
|
||||
a.initTracing(ctx)
|
||||
a.loadIndexPageTemplate()
|
||||
a.loadIndexPageTemplates()
|
||||
a.setHealthStatus()
|
||||
|
||||
a.log.Info(logs.SIGHUPConfigReloadCompleted)
|
||||
|
|
|
@ -62,8 +62,9 @@ const (
|
|||
|
||||
cfgReconnectInterval = "reconnect_interval"
|
||||
|
||||
cfgIndexPageEnabled = "index_page.enabled"
|
||||
cfgIndexPageTemplatePath = "index_page.template_path"
|
||||
cfgIndexPageEnabled = "index_page.enabled"
|
||||
cfgIndexPageS3TemplatePath = "index_page.template_path.s3"
|
||||
cfgIndexPageNativeTemplatePath = "index_page.template_path.native"
|
||||
|
||||
// Web.
|
||||
cfgWebReadBufferSize = "web.read_buffer_size"
|
||||
|
@ -207,7 +208,8 @@ func settings() *viper.Viper {
|
|||
v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold)
|
||||
|
||||
v.SetDefault(cfgIndexPageEnabled, false)
|
||||
v.SetDefault(cfgIndexPageTemplatePath, "")
|
||||
v.SetDefault(cfgIndexPageS3TemplatePath, "")
|
||||
v.SetDefault(cfgIndexPageNativeTemplatePath, "")
|
||||
|
||||
// frostfs:
|
||||
v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut)
|
||||
|
|
|
@ -351,15 +351,23 @@ resolve_bucket:
|
|||
|
||||
# `index_page` section
|
||||
|
||||
Parameters for index HTML-page output with S3-bucket or S3-subdir content for `Get object` request
|
||||
Parameters for index HTML-page output. Activates if `GetObject` request returns `not found`. Two
|
||||
index page modes available:
|
||||
|
||||
* `s3` mode uses tree service for listing objects,
|
||||
* `native` sends requests to nodes via native protocol.
|
||||
If request pass S3-bucket name instead of CID, `s3` mode will be used, otherwise `native`.
|
||||
|
||||
```yaml
|
||||
index_page:
|
||||
enabled: false
|
||||
template_path: ""
|
||||
template_path:
|
||||
s3: ""
|
||||
native: ""
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|-----------------|----------|---------------|---------------|---------------------------------------------------------------------------------|
|
||||
| `enabled` | `bool` | yes | `false` | Flag to enable index_page return if no object with specified S3-name was found. |
|
||||
| `template_path` | `string` | yes | `""` | Path to .gotmpl file with html template for index_page. |
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|------------------------|----------|---------------|---------------|----------------------------------------------------------------------------------------------|
|
||||
| `enabled` | `bool` | yes | `false` | Flag to enable index_page return if no object with specified S3-name was found. |
|
||||
| `template_path.s3` | `string` | yes | `""` | Path to .gotmpl file with html templates for S3 index_page (which is using tree service fr . |
|
||||
| `template_path.native` | `string` | yes | `""` | Path to .gotmpl file with html templates for native index_page |
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"html/template"
|
||||
"net/url"
|
||||
"sort"
|
||||
|
@ -10,6 +11,7 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/docker/go-units"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
|
@ -25,19 +27,58 @@ const (
|
|||
|
||||
type (
|
||||
BrowsePageData struct {
|
||||
BucketName,
|
||||
Prefix string
|
||||
Objects []ResponseObject
|
||||
BucketInfo *data.BucketInfo
|
||||
Prefix string
|
||||
Objects []ResponseObject
|
||||
}
|
||||
ResponseObject struct {
|
||||
OID string
|
||||
Created string
|
||||
FileName string
|
||||
FilePath string
|
||||
Size string
|
||||
IsDir bool
|
||||
}
|
||||
)
|
||||
|
||||
func newListObjectsResponseS3(attrs map[string]string) ResponseObject {
|
||||
return ResponseObject{
|
||||
OID: attrs[attrOID],
|
||||
Created: attrs[attrCreated],
|
||||
FileName: attrs[attrFileName],
|
||||
Size: attrs[attrSize],
|
||||
IsDir: attrs[attrOID] == "",
|
||||
}
|
||||
}
|
||||
|
||||
func newListObjectsResponseNative(attrs map[string]string) ResponseObject {
|
||||
return ResponseObject{
|
||||
OID: attrs[attrOID],
|
||||
Created: attrs[object.AttributeTimestamp] + "000",
|
||||
FileName: lastPathElement(attrs[object.AttributeFilePath]),
|
||||
FilePath: attrs[object.AttributeFilePath],
|
||||
Size: attrs[attrSize],
|
||||
IsDir: false,
|
||||
}
|
||||
}
|
||||
|
||||
func getNextDir(filepath, prefix string) string {
|
||||
restPath := strings.Replace(filepath, prefix, "", 1)
|
||||
index := strings.Index(restPath, "/")
|
||||
if index == -1 {
|
||||
return ""
|
||||
}
|
||||
return restPath[:index]
|
||||
}
|
||||
|
||||
func lastPathElement(path string) string {
|
||||
index := strings.LastIndex(path, "/")
|
||||
if index == len(path)-1 {
|
||||
index = strings.LastIndex(path[:index], "/")
|
||||
}
|
||||
return path[index+1:]
|
||||
}
|
||||
|
||||
func parseTimestamp(tstamp string) (time.Time, error) {
|
||||
millis, err := strconv.ParseInt(tstamp, 10, 64)
|
||||
if err != nil {
|
||||
|
@ -47,16 +88,6 @@ func parseTimestamp(tstamp string) (time.Time, error) {
|
|||
return time.UnixMilli(millis), nil
|
||||
}
|
||||
|
||||
func NewResponseObject(nodes map[string]string) ResponseObject {
|
||||
return ResponseObject{
|
||||
OID: nodes[attrOID],
|
||||
Created: nodes[attrCreated],
|
||||
FileName: nodes[attrFileName],
|
||||
Size: nodes[attrSize],
|
||||
IsDir: nodes[attrOID] == "",
|
||||
}
|
||||
}
|
||||
|
||||
func formatTimestamp(strdate string) string {
|
||||
date, err := parseTimestamp(strdate)
|
||||
if err != nil || date.IsZero() {
|
||||
|
@ -94,12 +125,9 @@ func trimPrefix(encPrefix string) string {
|
|||
return prefix[:slashIndex]
|
||||
}
|
||||
|
||||
func urlencode(prefix, filename string) string {
|
||||
func urlencode(path string) string {
|
||||
var res strings.Builder
|
||||
path := filename
|
||||
if prefix != "" {
|
||||
path = strings.Join([]string{prefix, filename}, "/")
|
||||
}
|
||||
|
||||
prefixParts := strings.Split(path, "/")
|
||||
for _, prefixPart := range prefixParts {
|
||||
prefixPart = "/" + url.PathEscape(prefixPart)
|
||||
|
@ -112,28 +140,32 @@ func urlencode(prefix, filename string) string {
|
|||
return res.String()
|
||||
}
|
||||
|
||||
func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) {
|
||||
log := h.log.With(zap.String("bucket", bucketInfo.Name))
|
||||
type browseParams struct {
|
||||
bucketInfo *data.BucketInfo
|
||||
template string
|
||||
listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) ([]ResponseObject, error)
|
||||
prefix string
|
||||
}
|
||||
|
||||
func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) {
|
||||
log := h.log.With(
|
||||
zap.String("bucket", p.bucketInfo.Name),
|
||||
zap.String("container", p.bucketInfo.CID.EncodeToString()),
|
||||
zap.String("prefix", p.prefix),
|
||||
)
|
||||
ctx := utils.GetContextFromRequest(c)
|
||||
nodes, err := h.listObjects(ctx, bucketInfo, prefix)
|
||||
objects, err := p.listObjects(ctx, p.bucketInfo, p.prefix)
|
||||
if err != nil {
|
||||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
}
|
||||
|
||||
respObjects := make([]ResponseObject, len(nodes))
|
||||
|
||||
for i, node := range nodes {
|
||||
respObjects[i] = NewResponseObject(node)
|
||||
}
|
||||
|
||||
sort.Slice(respObjects, func(i, j int) bool {
|
||||
if respObjects[i].IsDir == respObjects[j].IsDir {
|
||||
return respObjects[i].FileName < respObjects[j].FileName
|
||||
sort.Slice(objects, func(i, j int) bool {
|
||||
if objects[i].IsDir == objects[j].IsDir {
|
||||
return objects[i].FileName < objects[j].FileName
|
||||
}
|
||||
return respObjects[i].IsDir
|
||||
return objects[i].IsDir
|
||||
})
|
||||
indexTemplate := h.config.IndexPageTemplate()
|
||||
|
||||
tmpl, err := template.New("index").Funcs(template.FuncMap{
|
||||
"formatTimestamp": formatTimestamp,
|
||||
|
@ -141,15 +173,15 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketI
|
|||
"trimPrefix": trimPrefix,
|
||||
"urlencode": urlencode,
|
||||
"parentDir": parentDir,
|
||||
}).Parse(indexTemplate)
|
||||
}).Parse(p.template)
|
||||
if err != nil {
|
||||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
}
|
||||
if err = tmpl.Execute(c, &BrowsePageData{
|
||||
BucketName: bucketInfo.Name,
|
||||
Prefix: prefix,
|
||||
Objects: respObjects,
|
||||
BucketInfo: p.bucketInfo,
|
||||
Prefix: p.prefix,
|
||||
Objects: objects,
|
||||
}); err != nil {
|
||||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
|
|
|
@ -23,10 +23,9 @@ import (
|
|||
|
||||
// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
|
||||
func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
||||
test, _ := c.UserValue("oid").(string)
|
||||
var id oid.ID
|
||||
err := id.DecodeString(test)
|
||||
if err != nil {
|
||||
testCID, _ := c.UserValue("cid").(string)
|
||||
var cntID cid.ID
|
||||
if err := cntID.DecodeString(testCID); err != nil {
|
||||
h.byObjectName(c, h.receiveFile)
|
||||
} else {
|
||||
h.byAddress(c, h.receiveFile)
|
||||
|
@ -45,7 +44,7 @@ func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) {
|
|||
h.byAttribute(c, h.receiveFile)
|
||||
}
|
||||
|
||||
func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) {
|
||||
func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) {
|
||||
filters := object.NewSearchFilters()
|
||||
filters.AddRootFilter()
|
||||
filters.AddFilter(key, val, op)
|
||||
|
@ -54,7 +53,7 @@ func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op
|
|||
PrmAuth: PrmAuth{
|
||||
BearerToken: bearerToken(ctx),
|
||||
},
|
||||
Container: *cnrID,
|
||||
Container: cnrID,
|
||||
Filters: filters,
|
||||
}
|
||||
|
||||
|
@ -101,7 +100,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) {
|
|||
return
|
||||
}
|
||||
|
||||
resSearch, err := h.search(ctx, &bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
|
||||
resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
|
||||
if err != nil {
|
||||
log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
|
||||
response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
|
||||
|
|
|
@ -6,7 +6,9 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
|
@ -31,7 +33,8 @@ type Config interface {
|
|||
ZipCompression() bool
|
||||
ClientCut() bool
|
||||
IndexPageEnabled() bool
|
||||
IndexPageTemplate() string
|
||||
IndexPageS3Template() string
|
||||
IndexPageNativeTemplate() string
|
||||
BufferMaxSizeForPut() uint64
|
||||
NamespaceHeader() string
|
||||
}
|
||||
|
@ -182,7 +185,7 @@ func New(params *AppParams, config Config, tree *tree.Tree) *Handler {
|
|||
func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
||||
var (
|
||||
idCnr, _ = c.UserValue("cid").(string)
|
||||
idObj, _ = c.UserValue("oid").(string)
|
||||
idObj, _ = url.PathUnescape(c.UserValue("oid").(string))
|
||||
log = h.log.With(zap.String("cid", idCnr), zap.String("oid", idObj))
|
||||
)
|
||||
|
||||
|
@ -196,6 +199,18 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ
|
|||
|
||||
objID := new(oid.ID)
|
||||
if err = objID.DecodeString(idObj); err != nil {
|
||||
if h.config.IndexPageEnabled() {
|
||||
var addr oid.Address
|
||||
addr.SetContainer(bktInfo.CID)
|
||||
c.SetStatusCode(fasthttp.StatusNotFound)
|
||||
h.browseObjects(c, browseParams{
|
||||
bucketInfo: bktInfo,
|
||||
prefix: idObj,
|
||||
listObjects: h.getDirObjectsNative,
|
||||
template: h.config.IndexPageNativeTemplate(),
|
||||
})
|
||||
return
|
||||
}
|
||||
log.Error(logs.WrongObjectID, zap.Error(err))
|
||||
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
|
||||
return
|
||||
|
@ -205,6 +220,28 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ
|
|||
addr.SetContainer(bktInfo.CID)
|
||||
addr.SetObject(*objID)
|
||||
|
||||
_, err = h.frostfs.GetObject(ctx, PrmObjectGet{
|
||||
PrmAuth: PrmAuth{
|
||||
BearerToken: bearerToken(ctx),
|
||||
},
|
||||
Address: addr,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if h.config.IndexPageEnabled() {
|
||||
c.SetStatusCode(fasthttp.StatusNotFound)
|
||||
h.browseObjects(c, browseParams{
|
||||
bucketInfo: bktInfo,
|
||||
prefix: "",
|
||||
listObjects: h.getDirObjectsNative,
|
||||
template: h.config.IndexPageNativeTemplate(),
|
||||
})
|
||||
return
|
||||
}
|
||||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
}
|
||||
|
||||
f(ctx, *h.newRequest(c, log), addr)
|
||||
}
|
||||
|
||||
|
@ -237,7 +274,12 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r
|
|||
if isDir(unescapedKey) || isContainerRoot(unescapedKey) {
|
||||
if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK {
|
||||
c.SetStatusCode(code)
|
||||
h.browseObjects(c, bktInfo, unescapedKey)
|
||||
h.browseObjects(c, browseParams{
|
||||
bucketInfo: bktInfo,
|
||||
prefix: unescapedKey,
|
||||
listObjects: h.getDirObjectsS3,
|
||||
template: h.config.IndexPageS3Template(),
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
@ -294,7 +336,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re
|
|||
return
|
||||
}
|
||||
|
||||
res, err := h.search(ctx, &bktInfo.CID, key, val, object.MatchStringEqual)
|
||||
res, err := h.search(ctx, bktInfo.CID, key, val, object.MatchStringEqual)
|
||||
if err != nil {
|
||||
log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
|
||||
response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
|
||||
|
@ -391,24 +433,125 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket
|
|||
return bktInfo, err
|
||||
}
|
||||
|
||||
func (h *Handler) listObjects(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]map[string]string, error) {
|
||||
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) {
|
||||
nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var objects = make([]map[string]string, 0, len(nodes))
|
||||
var objects = make([]ResponseObject, 0, len(nodes))
|
||||
for _, node := range nodes {
|
||||
meta := node.GetMeta()
|
||||
if meta == nil {
|
||||
continue
|
||||
}
|
||||
var obj = make(map[string]string, len(meta))
|
||||
var attrs = make(map[string]string, len(meta))
|
||||
for _, m := range meta {
|
||||
obj[m.GetKey()] = string(m.GetValue())
|
||||
attrs[m.GetKey()] = string(m.GetValue())
|
||||
}
|
||||
obj := newListObjectsResponseS3(attrs)
|
||||
obj.FilePath = prefix + obj.FileName
|
||||
objects = append(objects, obj)
|
||||
}
|
||||
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) {
|
||||
basePath := strings.TrimRightFunc(prefix, func(r rune) bool {
|
||||
return r != '/'
|
||||
})
|
||||
objectIDs, err := h.search(ctx, bucketInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer objectIDs.Close()
|
||||
|
||||
objects, err := h.headDirObjects(ctx, bucketInfo.CID, objectIDs, basePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
func (h *Handler) headDirObjects(ctx context.Context, cID cid.ID, objectIDs ResObjectSearch, basePath string) ([]ResponseObject, error) {
|
||||
const initialSliceCapacity = 100
|
||||
|
||||
var (
|
||||
log = h.log.With(
|
||||
zap.String("cid", cID.EncodeToString()),
|
||||
zap.String("prefix", basePath),
|
||||
)
|
||||
mu = sync.Mutex{}
|
||||
wg = sync.WaitGroup{}
|
||||
errChan = make(chan error)
|
||||
|
||||
addr oid.Address
|
||||
objects = make([]ResponseObject, 0, initialSliceCapacity)
|
||||
dirs = sync.Map{}
|
||||
auth = PrmAuth{
|
||||
BearerToken: bearerToken(ctx),
|
||||
}
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
go func() {
|
||||
for err := range errChan {
|
||||
if err != nil {
|
||||
log.Error(logs.FailedToHeadObject, zap.Error(err))
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
addr.SetContainer(cID)
|
||||
err := objectIDs.Iterate(func(id oid.ID) bool {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
addr.SetObject(id)
|
||||
obj, err := h.frostfs.HeadObject(ctx, PrmObjectHead{
|
||||
PrmAuth: auth,
|
||||
Address: addr,
|
||||
})
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
|
||||
attrs := loadAttributes(obj.Attributes())
|
||||
attrs[attrOID] = id.EncodeToString()
|
||||
attrs[attrSize] = strconv.FormatUint(obj.PayloadSize(), 10)
|
||||
|
||||
dirname := getNextDir(attrs[object.AttributeFilePath], basePath)
|
||||
if dirname == "" {
|
||||
mu.Lock()
|
||||
objects = append(objects, newListObjectsResponseNative(attrs))
|
||||
mu.Unlock()
|
||||
} else if _, ok := dirs.Load(dirname); !ok {
|
||||
mu.Lock()
|
||||
objects = append(objects, ResponseObject{
|
||||
FileName: dirname,
|
||||
FilePath: basePath + dirname,
|
||||
IsDir: true,
|
||||
})
|
||||
mu.Unlock()
|
||||
dirs.Store(dirname, true)
|
||||
}
|
||||
}()
|
||||
|
||||
return false
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errChan)
|
||||
|
||||
return objects, nil
|
||||
}
|
||||
|
|
|
@ -57,10 +57,10 @@ func (c *configMock) IndexPageEnabled() bool {
|
|||
return false
|
||||
}
|
||||
|
||||
func (c *configMock) IndexPageTemplatePath() string {
|
||||
func (c *configMock) IndexPageS3Template() string {
|
||||
return ""
|
||||
}
|
||||
func (c *configMock) IndexPageTemplate() string {
|
||||
func (c *configMock) IndexPageNativeTemplate() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
|
|
|
@ -12,6 +12,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -59,6 +60,14 @@ func checkErrorType(err error) int {
|
|||
}
|
||||
}
|
||||
|
||||
func loadAttributes(attrs []object.Attribute) map[string]string {
|
||||
result := make(map[string]string)
|
||||
for _, attr := range attrs {
|
||||
result[attr.Key()] = attr.Value()
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func isValidToken(s string) bool {
|
||||
for _, c := range s {
|
||||
if c <= ' ' || c > 127 {
|
||||
|
|
|
@ -71,6 +71,7 @@ const (
|
|||
AddedStoragePeer = "added storage peer" // Info in ../../settings.go
|
||||
CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go
|
||||
CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go
|
||||
FailedToHeadObject = "failed to HEAD object" // Error in ../handler/handler.go
|
||||
InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go
|
||||
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
|
||||
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
{{$bucketName := .BucketName}}
|
||||
{{$bucketName := .BucketInfo.Name}}
|
||||
{{ $prefix := trimPrefix .Prefix }}
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
@ -42,7 +42,7 @@
|
|||
{{if $trimmedPrefix }}
|
||||
<tr>
|
||||
<td>
|
||||
⮐<a href="/get/{{$bucketName}}{{ urlencode $trimmedPrefix "" }}">..</a>
|
||||
⮐<a href="/get/{{$bucketName}}{{ urlencode $trimmedPrefix }}/">..</a>
|
||||
</td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
|
@ -63,12 +63,12 @@
|
|||
<td>
|
||||
{{if .IsDir}}
|
||||
🗀
|
||||
<a href="/get/{{ $bucketName }}{{ urlencode $prefix .FileName }}/">
|
||||
<a href="/get/{{ $bucketName }}{{ urlencode .FilePath }}/">
|
||||
{{.FileName}}/
|
||||
</a>
|
||||
{{else}}
|
||||
🗎
|
||||
<a href="/get/{{ $bucketName }}{{ urlencode $prefix .FileName }}">
|
||||
<a href="/get/{{ $bucketName }}{{ urlencode .FilePath }}">
|
||||
{{.FileName}}
|
||||
</a>
|
||||
{{end}}
|
||||
|
@ -77,7 +77,7 @@
|
|||
<td>{{if not .IsDir}}{{ formatTimestamp .Created }}{{end}}</td>
|
||||
<td>
|
||||
{{ if not .IsDir }}
|
||||
<a href="/get/{{ $bucketName}}{{ urlencode $prefix .FileName }}?download=true">
|
||||
<a href="/get/{{ $bucketName}}{{ urlencode .FilePath }}?download=true">
|
||||
Link
|
||||
</a>
|
||||
{{ end }}
|
||||
|
|
101
internal/templates/index_native.gotmpl
Normal file
101
internal/templates/index_native.gotmpl
Normal file
|
@ -0,0 +1,101 @@
|
|||
{{$container := .BucketInfo.CID}}
|
||||
{{ $prefix := trimPrefix .Prefix }}
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8"/>
|
||||
<title>Index of frostfs://{{$container}}/{{if $prefix}}/{{$prefix}}/{{end}}</title>
|
||||
<style>
|
||||
table {
|
||||
width: 80%;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
|
||||
body {
|
||||
background: #f2f2f2;
|
||||
}
|
||||
|
||||
table, th, td {
|
||||
border: 0 solid transparent;
|
||||
}
|
||||
|
||||
th, td {
|
||||
padding: 10px;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
th {
|
||||
background-color: #c3bcbc;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 1.5em;
|
||||
}
|
||||
|
||||
tr:nth-child(even) {
|
||||
background-color: #ebe7e7;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Index of frostfs://{{$container}}/{{if $prefix}}{{$prefix}}/{{end}}</h1>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Filename</th>
|
||||
<th>Size</th>
|
||||
<th>Created</th>
|
||||
<th>Download</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{ $trimmedPrefix := trimPrefix $prefix }}
|
||||
{{if $trimmedPrefix }}
|
||||
<tr>
|
||||
<td>
|
||||
⮐<a href="/get/{{$container}}{{ urlencode $trimmedPrefix }}/">..</a>
|
||||
</td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
</tr>
|
||||
{{else}}
|
||||
<tr>
|
||||
<td>
|
||||
⮐<a href="/get/{{ $container }}/">..</a>
|
||||
</td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
</tr>
|
||||
{{end}}
|
||||
{{range .Objects}}
|
||||
<tr>
|
||||
<td>
|
||||
{{if .IsDir}}
|
||||
🗀
|
||||
<a href="/get/{{ $container }}{{ urlencode .FilePath }}/">
|
||||
{{.FileName}}/
|
||||
</a>
|
||||
{{else}}
|
||||
🗎
|
||||
<a href="/get/{{ $container }}/{{ .OID }}">
|
||||
{{.FileName}}
|
||||
</a>
|
||||
{{end}}
|
||||
</td>
|
||||
<td>{{if .Size}}{{ formatSize .Size }}{{end}}</td>
|
||||
<td>{{if .Created}}{{ formatTimestamp .Created }}{{end}}</td>
|
||||
<td>
|
||||
{{ if .OID }}
|
||||
<a href="/get/{{ $container}}/{{ .OID }}?download=true">
|
||||
Link
|
||||
</a>
|
||||
{{ end }}
|
||||
</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
|
@ -4,3 +4,6 @@ import _ "embed"
|
|||
|
||||
//go:embed index.gotmpl
|
||||
var DefaultIndexTemplate string
|
||||
|
||||
//go:embed index_native.gotmpl
|
||||
var DefaultIndexNativeTemplate string
|
||||
|
|
Loading…
Reference in a new issue