[#151] index page: Add browse via native protocol
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
parent
495f745535
commit
e0a0f53fb8
10 changed files with 359 additions and 111 deletions
|
@ -218,20 +218,23 @@ func (a *app) loadIndexPageTemplate() {
|
||||||
if !a.settings.IndexPageEnabled() {
|
if !a.settings.IndexPageEnabled() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
reader, err := os.Open(a.cfg.GetString(cfgIndexPageTemplatePath))
|
path := a.cfg.GetString(cfgIndexPageTemplatePath)
|
||||||
|
tmpl, err := a.readTemplate(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.settings.setIndexTemplate("")
|
a.settings.setIndexTemplate("")
|
||||||
a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err))
|
a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err))
|
||||||
return
|
} else {
|
||||||
}
|
|
||||||
tmpl, err := io.ReadAll(reader)
|
|
||||||
if err != nil {
|
|
||||||
a.settings.setIndexTemplate("")
|
|
||||||
a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.settings.setIndexTemplate(string(tmpl))
|
a.settings.setIndexTemplate(string(tmpl))
|
||||||
a.log.Info(logs.SetCustomIndexPageTemplate)
|
a.log.Info(logs.SetCustomIndexPageTemplate, zap.String("path", path))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *app) readTemplate(path string) ([]byte, error) {
|
||||||
|
reader, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return io.ReadAll(reader)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *appSettings) ClientCut() bool {
|
func (s *appSettings) ClientCut() bool {
|
||||||
|
|
|
@ -206,9 +206,6 @@ func settings() *viper.Viper {
|
||||||
// pool:
|
// pool:
|
||||||
v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold)
|
v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold)
|
||||||
|
|
||||||
v.SetDefault(cfgIndexPageEnabled, false)
|
|
||||||
v.SetDefault(cfgIndexPageTemplatePath, "")
|
|
||||||
|
|
||||||
// frostfs:
|
// frostfs:
|
||||||
v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut)
|
v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut)
|
||||||
|
|
||||||
|
|
|
@ -351,7 +351,12 @@ resolve_bucket:
|
||||||
|
|
||||||
# `index_page` section
|
# `index_page` section
|
||||||
|
|
||||||
Parameters for index HTML-page output with S3-bucket or S3-subdir content for `Get object` request
|
Parameters for index HTML-page output. Activates if `GetObject` request returns `not found`. Two
|
||||||
|
index page modes available:
|
||||||
|
|
||||||
|
* `s3` mode uses tree service for listing objects,
|
||||||
|
* `native` sends requests to nodes via native protocol.
|
||||||
|
If request pass S3-bucket name instead of CID, `s3` mode will be used, otherwise `native`.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
index_page:
|
index_page:
|
||||||
|
|
|
@ -1,15 +1,21 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"html/template"
|
"html/template"
|
||||||
"net/url"
|
"net/url"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/docker/go-units"
|
"github.com/docker/go-units"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -25,19 +31,66 @@ const (
|
||||||
|
|
||||||
type (
|
type (
|
||||||
BrowsePageData struct {
|
BrowsePageData struct {
|
||||||
BucketName,
|
BucketInfo *data.BucketInfo
|
||||||
Prefix string
|
Prefix string
|
||||||
Objects []ResponseObject
|
Objects []ResponseObject
|
||||||
|
IsNative bool
|
||||||
}
|
}
|
||||||
ResponseObject struct {
|
ResponseObject struct {
|
||||||
OID string
|
OID string
|
||||||
Created string
|
Created string
|
||||||
FileName string
|
FileName string
|
||||||
|
FilePath string
|
||||||
Size string
|
Size string
|
||||||
IsDir bool
|
IsDir bool
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func newListObjectsResponseS3(attrs map[string]string) ResponseObject {
|
||||||
|
return ResponseObject{
|
||||||
|
OID: attrs[attrOID],
|
||||||
|
Created: attrs[attrCreated],
|
||||||
|
FileName: attrs[attrFileName],
|
||||||
|
Size: attrs[attrSize],
|
||||||
|
IsDir: attrs[attrOID] == "",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newListObjectsResponseNative(attrs map[string]string) ResponseObject {
|
||||||
|
filename := lastPathElement(attrs[object.AttributeFilePath])
|
||||||
|
if filename == "" {
|
||||||
|
filename = attrs[attrFileName]
|
||||||
|
}
|
||||||
|
return ResponseObject{
|
||||||
|
OID: attrs[attrOID],
|
||||||
|
Created: attrs[object.AttributeTimestamp] + "000",
|
||||||
|
FileName: filename,
|
||||||
|
FilePath: attrs[object.AttributeFilePath],
|
||||||
|
Size: attrs[attrSize],
|
||||||
|
IsDir: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func getNextDir(filepath, prefix string) string {
|
||||||
|
restPath := strings.Replace(filepath, prefix, "", 1)
|
||||||
|
index := strings.Index(restPath, "/")
|
||||||
|
if index == -1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return restPath[:index]
|
||||||
|
}
|
||||||
|
|
||||||
|
func lastPathElement(path string) string {
|
||||||
|
if path == "" {
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
index := strings.LastIndex(path, "/")
|
||||||
|
if index == len(path)-1 {
|
||||||
|
index = strings.LastIndex(path[:index], "/")
|
||||||
|
}
|
||||||
|
return path[index+1:]
|
||||||
|
}
|
||||||
|
|
||||||
func parseTimestamp(tstamp string) (time.Time, error) {
|
func parseTimestamp(tstamp string) (time.Time, error) {
|
||||||
millis, err := strconv.ParseInt(tstamp, 10, 64)
|
millis, err := strconv.ParseInt(tstamp, 10, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -47,16 +100,6 @@ func parseTimestamp(tstamp string) (time.Time, error) {
|
||||||
return time.UnixMilli(millis), nil
|
return time.UnixMilli(millis), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewResponseObject(nodes map[string]string) ResponseObject {
|
|
||||||
return ResponseObject{
|
|
||||||
OID: nodes[attrOID],
|
|
||||||
Created: nodes[attrCreated],
|
|
||||||
FileName: nodes[attrFileName],
|
|
||||||
Size: nodes[attrSize],
|
|
||||||
IsDir: nodes[attrOID] == "",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func formatTimestamp(strdate string) string {
|
func formatTimestamp(strdate string) string {
|
||||||
date, err := parseTimestamp(strdate)
|
date, err := parseTimestamp(strdate)
|
||||||
if err != nil || date.IsZero() {
|
if err != nil || date.IsZero() {
|
||||||
|
@ -94,12 +137,9 @@ func trimPrefix(encPrefix string) string {
|
||||||
return prefix[:slashIndex]
|
return prefix[:slashIndex]
|
||||||
}
|
}
|
||||||
|
|
||||||
func urlencode(prefix, filename string) string {
|
func urlencode(path string) string {
|
||||||
var res strings.Builder
|
var res strings.Builder
|
||||||
path := filename
|
|
||||||
if prefix != "" {
|
|
||||||
path = strings.Join([]string{prefix, filename}, "/")
|
|
||||||
}
|
|
||||||
prefixParts := strings.Split(path, "/")
|
prefixParts := strings.Split(path, "/")
|
||||||
for _, prefixPart := range prefixParts {
|
for _, prefixPart := range prefixParts {
|
||||||
prefixPart = "/" + url.PathEscape(prefixPart)
|
prefixPart = "/" + url.PathEscape(prefixPart)
|
||||||
|
@ -112,28 +152,185 @@ func urlencode(prefix, filename string) string {
|
||||||
return res.String()
|
return res.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) {
|
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) {
|
||||||
log := h.log.With(zap.String("bucket", bucketInfo.Name))
|
nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var objects = make([]ResponseObject, 0, len(nodes))
|
||||||
|
for _, node := range nodes {
|
||||||
|
meta := node.GetMeta()
|
||||||
|
if meta == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var attrs = make(map[string]string, len(meta))
|
||||||
|
for _, m := range meta {
|
||||||
|
attrs[m.GetKey()] = string(m.GetValue())
|
||||||
|
}
|
||||||
|
obj := newListObjectsResponseS3(attrs)
|
||||||
|
obj.FilePath = prefix + obj.FileName
|
||||||
|
objects = append(objects, obj)
|
||||||
|
}
|
||||||
|
|
||||||
|
return objects, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type headDirParams struct {
|
||||||
|
cnrID cid.ID
|
||||||
|
objectIDs ResObjectSearch
|
||||||
|
basePath string
|
||||||
|
objCh chan<- ResponseObject
|
||||||
|
errCh chan<- error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) {
|
||||||
|
const initialSliceCapacity = 100
|
||||||
|
|
||||||
|
log := h.log.With(
|
||||||
|
zap.String("cid", bucketInfo.CID.EncodeToString()),
|
||||||
|
zap.String("prefix", prefix),
|
||||||
|
)
|
||||||
|
basePath := strings.TrimRightFunc(prefix, func(r rune) bool {
|
||||||
|
return r != '/'
|
||||||
|
})
|
||||||
|
filters := []object.SearchMatchType{object.MatchCommonPrefix}
|
||||||
|
if basePath == "" {
|
||||||
|
filters = append(filters, object.MatchNotPresent)
|
||||||
|
}
|
||||||
|
objCh := make(chan ResponseObject)
|
||||||
|
errCh := make(chan error)
|
||||||
|
done := make(chan struct{})
|
||||||
|
objects := make([]ResponseObject, 0, initialSliceCapacity)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
for err := range errCh {
|
||||||
|
if err != nil {
|
||||||
|
log.Error(logs.FailedToHeadObject, zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
done <- struct{}{}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
for obj := range objCh {
|
||||||
|
objects = append(objects, obj)
|
||||||
|
}
|
||||||
|
done <- struct{}{}
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg := sync.WaitGroup{}
|
||||||
|
for _, filter := range filters {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(filter object.SearchMatchType) {
|
||||||
|
defer wg.Done()
|
||||||
|
objectIDs, err := h.search(ctx, bucketInfo.CID, object.AttributeFilePath, prefix, filter)
|
||||||
|
if err != nil {
|
||||||
|
errCh <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer objectIDs.Close()
|
||||||
|
h.headDirObjects(ctx, headDirParams{
|
||||||
|
cnrID: bucketInfo.CID,
|
||||||
|
objectIDs: objectIDs,
|
||||||
|
basePath: basePath,
|
||||||
|
objCh: objCh,
|
||||||
|
errCh: errCh,
|
||||||
|
})
|
||||||
|
}(filter)
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
close(errCh)
|
||||||
|
close(objCh)
|
||||||
|
<-done
|
||||||
|
<-done
|
||||||
|
|
||||||
|
return objects, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) headDirObjects(ctx context.Context, p headDirParams) {
|
||||||
|
wg := sync.WaitGroup{}
|
||||||
|
dirs := sync.Map{}
|
||||||
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err := p.objectIDs.Iterate(func(id oid.ID) bool {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(id oid.ID) {
|
||||||
|
defer wg.Done()
|
||||||
|
h.headDirObject(ctx, id, p, &dirs, cancel)
|
||||||
|
}(id)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
p.errCh <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) headDirObject(ctx context.Context, id oid.ID, p headDirParams, dirs *sync.Map, cancel context.CancelFunc) {
|
||||||
|
addr := newAddress(p.cnrID, id)
|
||||||
|
obj, err := h.frostfs.HeadObject(ctx, PrmObjectHead{
|
||||||
|
PrmAuth: PrmAuth{BearerToken: bearerToken(ctx)},
|
||||||
|
Address: addr,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
p.errCh <- err
|
||||||
|
cancel()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
attrs := loadAttributes(obj.Attributes())
|
||||||
|
attrs[attrOID] = id.EncodeToString()
|
||||||
|
attrs[attrSize] = strconv.FormatUint(obj.PayloadSize(), 10)
|
||||||
|
|
||||||
|
dirname := getNextDir(attrs[object.AttributeFilePath], p.basePath)
|
||||||
|
if dirname == "" {
|
||||||
|
p.objCh <- newListObjectsResponseNative(attrs)
|
||||||
|
} else if _, ok := dirs.Load(dirname); !ok {
|
||||||
|
p.objCh <- ResponseObject{
|
||||||
|
FileName: dirname,
|
||||||
|
FilePath: p.basePath + dirname,
|
||||||
|
IsDir: true,
|
||||||
|
}
|
||||||
|
dirs.Store(dirname, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type browseParams struct {
|
||||||
|
bucketInfo *data.BucketInfo
|
||||||
|
prefix string
|
||||||
|
isNative bool
|
||||||
|
listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) ([]ResponseObject, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) {
|
||||||
|
log := h.log.With(
|
||||||
|
zap.String("bucket", p.bucketInfo.Name),
|
||||||
|
zap.String("container", p.bucketInfo.CID.EncodeToString()),
|
||||||
|
zap.String("prefix", p.prefix),
|
||||||
|
)
|
||||||
ctx := utils.GetContextFromRequest(c)
|
ctx := utils.GetContextFromRequest(c)
|
||||||
nodes, err := h.listObjects(ctx, bucketInfo, prefix)
|
objects, err := p.listObjects(ctx, p.bucketInfo, p.prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logAndSendBucketError(c, log, err)
|
logAndSendBucketError(c, log, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
respObjects := make([]ResponseObject, len(nodes))
|
sort.Slice(objects, func(i, j int) bool {
|
||||||
|
if objects[i].IsDir == objects[j].IsDir {
|
||||||
for i, node := range nodes {
|
return objects[i].FileName < objects[j].FileName
|
||||||
respObjects[i] = NewResponseObject(node)
|
|
||||||
}
|
}
|
||||||
|
return objects[i].IsDir
|
||||||
sort.Slice(respObjects, func(i, j int) bool {
|
|
||||||
if respObjects[i].IsDir == respObjects[j].IsDir {
|
|
||||||
return respObjects[i].FileName < respObjects[j].FileName
|
|
||||||
}
|
|
||||||
return respObjects[i].IsDir
|
|
||||||
})
|
})
|
||||||
indexTemplate := h.config.IndexPageTemplate()
|
|
||||||
|
|
||||||
tmpl, err := template.New("index").Funcs(template.FuncMap{
|
tmpl, err := template.New("index").Funcs(template.FuncMap{
|
||||||
"formatTimestamp": formatTimestamp,
|
"formatTimestamp": formatTimestamp,
|
||||||
|
@ -141,15 +338,16 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketI
|
||||||
"trimPrefix": trimPrefix,
|
"trimPrefix": trimPrefix,
|
||||||
"urlencode": urlencode,
|
"urlencode": urlencode,
|
||||||
"parentDir": parentDir,
|
"parentDir": parentDir,
|
||||||
}).Parse(indexTemplate)
|
}).Parse(h.config.IndexPageTemplate())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logAndSendBucketError(c, log, err)
|
logAndSendBucketError(c, log, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = tmpl.Execute(c, &BrowsePageData{
|
if err = tmpl.Execute(c, &BrowsePageData{
|
||||||
BucketName: bucketInfo.Name,
|
BucketInfo: p.bucketInfo,
|
||||||
Prefix: prefix,
|
Prefix: p.prefix,
|
||||||
Objects: respObjects,
|
IsNative: p.isNative,
|
||||||
|
Objects: objects,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
logAndSendBucketError(c, log, err)
|
logAndSendBucketError(c, log, err)
|
||||||
return
|
return
|
||||||
|
|
|
@ -23,10 +23,9 @@ import (
|
||||||
|
|
||||||
// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
|
// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
|
||||||
func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
||||||
test, _ := c.UserValue("oid").(string)
|
cnrIDStr, _ := c.UserValue("cid").(string)
|
||||||
var id oid.ID
|
var cnrID cid.ID
|
||||||
err := id.DecodeString(test)
|
if err := cnrID.DecodeString(cnrIDStr); err != nil {
|
||||||
if err != nil {
|
|
||||||
h.byObjectName(c, h.receiveFile)
|
h.byObjectName(c, h.receiveFile)
|
||||||
} else {
|
} else {
|
||||||
h.byAddress(c, h.receiveFile)
|
h.byAddress(c, h.receiveFile)
|
||||||
|
@ -45,7 +44,7 @@ func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) {
|
||||||
h.byAttribute(c, h.receiveFile)
|
h.byAttribute(c, h.receiveFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) {
|
func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) {
|
||||||
filters := object.NewSearchFilters()
|
filters := object.NewSearchFilters()
|
||||||
filters.AddRootFilter()
|
filters.AddRootFilter()
|
||||||
filters.AddFilter(key, val, op)
|
filters.AddFilter(key, val, op)
|
||||||
|
@ -54,7 +53,7 @@ func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op
|
||||||
PrmAuth: PrmAuth{
|
PrmAuth: PrmAuth{
|
||||||
BearerToken: bearerToken(ctx),
|
BearerToken: bearerToken(ctx),
|
||||||
},
|
},
|
||||||
Container: *cnrID,
|
Container: cnrID,
|
||||||
Filters: filters,
|
Filters: filters,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -101,7 +100,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resSearch, err := h.search(ctx, &bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
|
resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
|
log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
|
||||||
response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
|
response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
|
||||||
|
|
|
@ -192,7 +192,7 @@ func New(params *AppParams, config Config, tree *tree.Tree) *Handler {
|
||||||
func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
||||||
var (
|
var (
|
||||||
idCnr, _ = c.UserValue("cid").(string)
|
idCnr, _ = c.UserValue("cid").(string)
|
||||||
idObj, _ = c.UserValue("oid").(string)
|
idObj, _ = url.PathUnescape(c.UserValue("oid").(string))
|
||||||
log = h.log.With(zap.String("cid", idCnr), zap.String("oid", idObj))
|
log = h.log.With(zap.String("cid", idCnr), zap.String("oid", idObj))
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -206,6 +206,18 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ
|
||||||
|
|
||||||
objID := new(oid.ID)
|
objID := new(oid.ID)
|
||||||
if err = objID.DecodeString(idObj); err != nil {
|
if err = objID.DecodeString(idObj); err != nil {
|
||||||
|
if h.config.IndexPageEnabled() {
|
||||||
|
var addr oid.Address
|
||||||
|
addr.SetContainer(bktInfo.CID)
|
||||||
|
c.SetStatusCode(fasthttp.StatusNotFound)
|
||||||
|
h.browseObjects(c, browseParams{
|
||||||
|
bucketInfo: bktInfo,
|
||||||
|
prefix: idObj,
|
||||||
|
listObjects: h.getDirObjectsNative,
|
||||||
|
isNative: true,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
log.Error(logs.WrongObjectID, zap.Error(err))
|
log.Error(logs.WrongObjectID, zap.Error(err))
|
||||||
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
|
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
|
@ -245,7 +257,12 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r
|
||||||
if isDir(unescapedKey) || isContainerRoot(unescapedKey) {
|
if isDir(unescapedKey) || isContainerRoot(unescapedKey) {
|
||||||
if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK {
|
if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK {
|
||||||
c.SetStatusCode(code)
|
c.SetStatusCode(code)
|
||||||
h.browseObjects(c, bktInfo, unescapedKey)
|
h.browseObjects(c, browseParams{
|
||||||
|
bucketInfo: bktInfo,
|
||||||
|
prefix: unescapedKey,
|
||||||
|
listObjects: h.getDirObjectsS3,
|
||||||
|
isNative: false,
|
||||||
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -299,7 +316,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := h.search(ctx, &bktInfo.CID, key, val, object.MatchStringEqual)
|
res, err := h.search(ctx, bktInfo.CID, key, val, object.MatchStringEqual)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
|
log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
|
||||||
response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
|
response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
|
||||||
|
@ -395,25 +412,3 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket
|
||||||
|
|
||||||
return bktInfo, err
|
return bktInfo, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) listObjects(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]map[string]string, error) {
|
|
||||||
nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var objects = make([]map[string]string, 0, len(nodes))
|
|
||||||
for _, node := range nodes {
|
|
||||||
meta := node.GetMeta()
|
|
||||||
if meta == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var obj = make(map[string]string, len(meta))
|
|
||||||
for _, m := range meta {
|
|
||||||
obj[m.GetKey()] = string(m.GetValue())
|
|
||||||
}
|
|
||||||
objects = append(objects, obj)
|
|
||||||
}
|
|
||||||
|
|
||||||
return objects, nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -57,10 +57,10 @@ func (c *configMock) IndexPageEnabled() bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *configMock) IndexPageTemplatePath() string {
|
func (c *configMock) IndexPageTemplate() string {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
func (c *configMock) IndexPageTemplate() string {
|
func (c *configMock) IndexPageNativeTemplate() string {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -61,6 +62,14 @@ func checkErrorType(err error) int {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadAttributes(attrs []object.Attribute) map[string]string {
|
||||||
|
result := make(map[string]string)
|
||||||
|
for _, attr := range attrs {
|
||||||
|
result[attr.Key()] = attr.Value()
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func isValidToken(s string) bool {
|
func isValidToken(s string) bool {
|
||||||
for _, c := range s {
|
for _, c := range s {
|
||||||
if c <= ' ' || c > 127 {
|
if c <= ' ' || c > 127 {
|
||||||
|
|
|
@ -71,6 +71,7 @@ const (
|
||||||
AddedStoragePeer = "added storage peer" // Info in ../../settings.go
|
AddedStoragePeer = "added storage peer" // Info in ../../settings.go
|
||||||
CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go
|
CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go
|
||||||
CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go
|
CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go
|
||||||
|
FailedToHeadObject = "failed to HEAD object" // Error in ../handler/handler.go
|
||||||
InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go
|
InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go
|
||||||
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
|
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
|
||||||
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
|
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
|
||||||
|
|
|
@ -1,10 +1,15 @@
|
||||||
{{$bucketName := .BucketName}}
|
{{$isNative := .IsNative}}
|
||||||
|
{{$container := .BucketInfo.Name}}
|
||||||
|
{{if $isNative}}
|
||||||
|
{{$container = .BucketInfo.CID}}
|
||||||
|
{{end}}
|
||||||
{{ $prefix := trimPrefix .Prefix }}
|
{{ $prefix := trimPrefix .Prefix }}
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8"/>
|
<meta charset="UTF-8"/>
|
||||||
<title>Index of s3://{{$bucketName}}/{{if $prefix}}/{{$prefix}}/{{end}}</title>
|
<title>Index of {{if $isNative}}frostfs{{else}}s3{{end}}://{{$container}}
|
||||||
|
/{{if $prefix}}/{{$prefix}}/{{end}}</title>
|
||||||
<style>
|
<style>
|
||||||
table {
|
table {
|
||||||
width: 80%;
|
width: 80%;
|
||||||
|
@ -23,15 +28,21 @@
|
||||||
th {
|
th {
|
||||||
background-color: #c3bcbc;
|
background-color: #c3bcbc;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
font-size: 1.5em;
|
||||||
|
}
|
||||||
tr:nth-child(even) {background-color: #ebe7e7;}
|
tr:nth-child(even) {background-color: #ebe7e7;}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<h1>Index of s3://{{$bucketName}}/{{if $prefix}}{{$prefix}}/{{end}}</h1>
|
<h1>Index of {{if $isNative}}frostfs{{else}}s3{{end}}://{{$container}}
|
||||||
|
/{{if $prefix}}{{$prefix}}/{{end}}</h1>
|
||||||
<table>
|
<table>
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
<th>Filename</th>
|
<th>Filename</th>
|
||||||
|
<th>OID</th>
|
||||||
<th>Size</th>
|
<th>Size</th>
|
||||||
<th>Created</th>
|
<th>Created</th>
|
||||||
<th>Download</th>
|
<th>Download</th>
|
||||||
|
@ -42,33 +53,62 @@
|
||||||
{{if $trimmedPrefix }}
|
{{if $trimmedPrefix }}
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>
|
||||||
⮐<a href="/get/{{$bucketName}}{{ urlencode $trimmedPrefix "" }}">..</a>
|
⮐<a href="/get/{{$container}}{{ urlencode $trimmedPrefix }}/">..</a>
|
||||||
</td>
|
</td>
|
||||||
<td></td>
|
<td></td>
|
||||||
<td></td>
|
<td></td>
|
||||||
<td></td>
|
<td></td>
|
||||||
|
<td></td>
|
||||||
</tr>
|
</tr>
|
||||||
{{else}}
|
{{else}}
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>
|
||||||
⮐<a href="/get/{{ $bucketName }}/">..</a>
|
⮐<a href="/get/{{$container}}/">..</a>
|
||||||
</td>
|
</td>
|
||||||
<td></td>
|
<td></td>
|
||||||
<td></td>
|
<td></td>
|
||||||
<td></td>
|
<td></td>
|
||||||
|
<td></td>
|
||||||
</tr>
|
</tr>
|
||||||
{{end}}
|
{{end}}
|
||||||
{{range .Objects}}
|
{{range .Objects}}
|
||||||
|
{{if $isNative}}
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>
|
||||||
{{if .IsDir}}
|
{{if .IsDir}}
|
||||||
🗀
|
🗀
|
||||||
<a href="/get/{{ $bucketName }}{{ urlencode $prefix .FileName }}/">
|
<a href="/get/{{$container}}{{ urlencode .FilePath }}/">
|
||||||
{{.FileName}}/
|
{{.FileName}}/
|
||||||
</a>
|
</a>
|
||||||
{{else}}
|
{{else}}
|
||||||
🗎
|
🗎
|
||||||
<a href="/get/{{ $bucketName }}{{ urlencode $prefix .FileName }}">
|
<a href="/get/{{$container}}/{{ .OID }}">
|
||||||
|
{{.FileName}}
|
||||||
|
</a>
|
||||||
|
{{end}}
|
||||||
|
</td>
|
||||||
|
<td>{{.OID}}</td>
|
||||||
|
<td>{{if .Size}}{{ formatSize .Size }}{{end}}</td>
|
||||||
|
<td>{{if .Created}}{{ formatTimestamp .Created }}{{end}}</td>
|
||||||
|
<td>
|
||||||
|
{{ if .OID }}
|
||||||
|
<a href="/get/{{$container}}/{{ .OID }}?download=true">
|
||||||
|
Link
|
||||||
|
</a>
|
||||||
|
{{ end }}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{{else}}
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
{{if .IsDir}}
|
||||||
|
🗀
|
||||||
|
<a href="/get/{{$container}}{{ urlencode .FilePath }}/">
|
||||||
|
{{.FileName}}/
|
||||||
|
</a>
|
||||||
|
{{else}}
|
||||||
|
🗎
|
||||||
|
<a href="/get/{{$container}}{{ urlencode .FilePath }}">
|
||||||
{{.FileName}}
|
{{.FileName}}
|
||||||
</a>
|
</a>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
@ -77,13 +117,14 @@
|
||||||
<td>{{if not .IsDir}}{{ formatTimestamp .Created }}{{end}}</td>
|
<td>{{if not .IsDir}}{{ formatTimestamp .Created }}{{end}}</td>
|
||||||
<td>
|
<td>
|
||||||
{{ if not .IsDir }}
|
{{ if not .IsDir }}
|
||||||
<a href="/get/{{ $bucketName}}{{ urlencode $prefix .FileName }}?download=true">
|
<a href="/get/{{$container}}{{ urlencode .FilePath }}?download=true">
|
||||||
Link
|
Link
|
||||||
</a>
|
</a>
|
||||||
{{ end }}
|
{{ end }}
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
{{end}}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</body>
|
</body>
|
||||||
|
|
Loading…
Reference in a new issue