355 lines
8 KiB
Go
355 lines
8 KiB
Go
package handler
|
|
|
|
import (
|
|
"context"
|
|
"html/template"
|
|
"net/url"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
"github.com/docker/go-units"
|
|
"github.com/valyala/fasthttp"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
const (
|
|
dateFormat = "02-01-2006 15:04"
|
|
attrOID = "OID"
|
|
attrCreated = "Created"
|
|
attrFileName = "FileName"
|
|
attrSize = "Size"
|
|
)
|
|
|
|
type (
|
|
BrowsePageData struct {
|
|
BucketInfo *data.BucketInfo
|
|
Prefix string
|
|
Objects []ResponseObject
|
|
IsNative bool
|
|
}
|
|
ResponseObject struct {
|
|
OID string
|
|
Created string
|
|
FileName string
|
|
FilePath string
|
|
Size string
|
|
IsDir bool
|
|
}
|
|
)
|
|
|
|
func newListObjectsResponseS3(attrs map[string]string) ResponseObject {
|
|
return ResponseObject{
|
|
OID: attrs[attrOID],
|
|
Created: attrs[attrCreated],
|
|
FileName: attrs[attrFileName],
|
|
Size: attrs[attrSize],
|
|
IsDir: attrs[attrOID] == "",
|
|
}
|
|
}
|
|
|
|
func newListObjectsResponseNative(attrs map[string]string) ResponseObject {
|
|
filename := lastPathElement(attrs[object.AttributeFilePath])
|
|
if filename == "" {
|
|
filename = attrs[attrFileName]
|
|
}
|
|
return ResponseObject{
|
|
OID: attrs[attrOID],
|
|
Created: attrs[object.AttributeTimestamp] + "000",
|
|
FileName: filename,
|
|
FilePath: attrs[object.AttributeFilePath],
|
|
Size: attrs[attrSize],
|
|
IsDir: false,
|
|
}
|
|
}
|
|
|
|
func getNextDir(filepath, prefix string) string {
|
|
restPath := strings.Replace(filepath, prefix, "", 1)
|
|
index := strings.Index(restPath, "/")
|
|
if index == -1 {
|
|
return ""
|
|
}
|
|
return restPath[:index]
|
|
}
|
|
|
|
func lastPathElement(path string) string {
|
|
if path == "" {
|
|
return path
|
|
}
|
|
index := strings.LastIndex(path, "/")
|
|
if index == len(path)-1 {
|
|
index = strings.LastIndex(path[:index], "/")
|
|
}
|
|
return path[index+1:]
|
|
}
|
|
|
|
func parseTimestamp(tstamp string) (time.Time, error) {
|
|
millis, err := strconv.ParseInt(tstamp, 10, 64)
|
|
if err != nil {
|
|
return time.Time{}, err
|
|
}
|
|
|
|
return time.UnixMilli(millis), nil
|
|
}
|
|
|
|
func formatTimestamp(strdate string) string {
|
|
date, err := parseTimestamp(strdate)
|
|
if err != nil || date.IsZero() {
|
|
return ""
|
|
}
|
|
|
|
return date.Format(dateFormat)
|
|
}
|
|
|
|
func formatSize(strsize string) string {
|
|
size, err := strconv.ParseFloat(strsize, 64)
|
|
if err != nil {
|
|
return "0B"
|
|
}
|
|
return units.HumanSize(size)
|
|
}
|
|
|
|
func parentDir(prefix string) string {
|
|
index := strings.LastIndex(prefix, "/")
|
|
if index == -1 {
|
|
return prefix
|
|
}
|
|
return prefix[index:]
|
|
}
|
|
|
|
func trimPrefix(encPrefix string) string {
|
|
prefix, err := url.PathUnescape(encPrefix)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
slashIndex := strings.LastIndex(prefix, "/")
|
|
if slashIndex == -1 {
|
|
return ""
|
|
}
|
|
return prefix[:slashIndex]
|
|
}
|
|
|
|
func urlencode(path string) string {
|
|
var res strings.Builder
|
|
|
|
prefixParts := strings.Split(path, "/")
|
|
for _, prefixPart := range prefixParts {
|
|
prefixPart = "/" + url.PathEscape(prefixPart)
|
|
if prefixPart == "/." || prefixPart == "/.." {
|
|
prefixPart = url.PathEscape(prefixPart)
|
|
}
|
|
res.WriteString(prefixPart)
|
|
}
|
|
|
|
return res.String()
|
|
}
|
|
|
|
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) {
|
|
nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var objects = make([]ResponseObject, 0, len(nodes))
|
|
for _, node := range nodes {
|
|
meta := node.GetMeta()
|
|
if meta == nil {
|
|
continue
|
|
}
|
|
var attrs = make(map[string]string, len(meta))
|
|
for _, m := range meta {
|
|
attrs[m.GetKey()] = string(m.GetValue())
|
|
}
|
|
obj := newListObjectsResponseS3(attrs)
|
|
obj.FilePath = prefix + obj.FileName
|
|
objects = append(objects, obj)
|
|
}
|
|
|
|
return objects, nil
|
|
}
|
|
|
|
type headDirParams struct {
|
|
cnrID cid.ID
|
|
objectIDs ResObjectSearch
|
|
basePath string
|
|
objCh chan<- ResponseObject
|
|
errCh chan<- error
|
|
}
|
|
|
|
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) {
|
|
const initialSliceCapacity = 100
|
|
|
|
log := h.log.With(
|
|
zap.String("cid", bucketInfo.CID.EncodeToString()),
|
|
zap.String("prefix", prefix),
|
|
)
|
|
basePath := strings.TrimRightFunc(prefix, func(r rune) bool {
|
|
return r != '/'
|
|
})
|
|
filters := []object.SearchMatchType{object.MatchCommonPrefix}
|
|
if basePath == "" {
|
|
filters = append(filters, object.MatchNotPresent)
|
|
}
|
|
objCh := make(chan ResponseObject)
|
|
errCh := make(chan error)
|
|
done := make(chan struct{})
|
|
objects := make([]ResponseObject, 0, initialSliceCapacity)
|
|
|
|
go func() {
|
|
for err := range errCh {
|
|
if err != nil {
|
|
log.Error(logs.FailedToHeadObject, zap.Error(err))
|
|
}
|
|
}
|
|
done <- struct{}{}
|
|
}()
|
|
go func() {
|
|
for obj := range objCh {
|
|
objects = append(objects, obj)
|
|
}
|
|
done <- struct{}{}
|
|
}()
|
|
|
|
wg := sync.WaitGroup{}
|
|
for _, filter := range filters {
|
|
wg.Add(1)
|
|
go func(filter object.SearchMatchType) {
|
|
defer wg.Done()
|
|
objectIDs, err := h.search(ctx, bucketInfo.CID, object.AttributeFilePath, prefix, filter)
|
|
if err != nil {
|
|
errCh <- err
|
|
return
|
|
}
|
|
defer objectIDs.Close()
|
|
h.headDirObjects(ctx, headDirParams{
|
|
cnrID: bucketInfo.CID,
|
|
objectIDs: objectIDs,
|
|
basePath: basePath,
|
|
objCh: objCh,
|
|
errCh: errCh,
|
|
})
|
|
}(filter)
|
|
}
|
|
|
|
wg.Wait()
|
|
close(errCh)
|
|
close(objCh)
|
|
<-done
|
|
<-done
|
|
|
|
return objects, nil
|
|
}
|
|
|
|
func (h *Handler) headDirObjects(ctx context.Context, p headDirParams) {
|
|
wg := sync.WaitGroup{}
|
|
dirs := sync.Map{}
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
|
|
err := p.objectIDs.Iterate(func(id oid.ID) bool {
|
|
wg.Add(1)
|
|
go func(id oid.ID) {
|
|
defer wg.Done()
|
|
h.headDirObject(ctx, id, p, &dirs, cancel)
|
|
}(id)
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
})
|
|
wg.Wait()
|
|
|
|
if err != nil {
|
|
p.errCh <- err
|
|
return
|
|
}
|
|
}
|
|
|
|
func (h *Handler) headDirObject(ctx context.Context, id oid.ID, p headDirParams, dirs *sync.Map, cancel context.CancelFunc) {
|
|
addr := newAddress(p.cnrID, id)
|
|
obj, err := h.frostfs.HeadObject(ctx, PrmObjectHead{
|
|
PrmAuth: PrmAuth{BearerToken: bearerToken(ctx)},
|
|
Address: addr,
|
|
})
|
|
if err != nil {
|
|
p.errCh <- err
|
|
cancel()
|
|
return
|
|
}
|
|
|
|
attrs := loadAttributes(obj.Attributes())
|
|
attrs[attrOID] = id.EncodeToString()
|
|
attrs[attrSize] = strconv.FormatUint(obj.PayloadSize(), 10)
|
|
|
|
dirname := getNextDir(attrs[object.AttributeFilePath], p.basePath)
|
|
if dirname == "" {
|
|
p.objCh <- newListObjectsResponseNative(attrs)
|
|
} else if _, ok := dirs.Load(dirname); !ok {
|
|
p.objCh <- ResponseObject{
|
|
FileName: dirname,
|
|
FilePath: p.basePath + dirname,
|
|
IsDir: true,
|
|
}
|
|
dirs.Store(dirname, true)
|
|
}
|
|
}
|
|
|
|
type browseParams struct {
|
|
bucketInfo *data.BucketInfo
|
|
prefix string
|
|
isNative bool
|
|
listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) ([]ResponseObject, error)
|
|
}
|
|
|
|
func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) {
|
|
log := h.log.With(
|
|
zap.String("bucket", p.bucketInfo.Name),
|
|
zap.String("container", p.bucketInfo.CID.EncodeToString()),
|
|
zap.String("prefix", p.prefix),
|
|
)
|
|
ctx := utils.GetContextFromRequest(c)
|
|
objects, err := p.listObjects(ctx, p.bucketInfo, p.prefix)
|
|
if err != nil {
|
|
logAndSendBucketError(c, log, err)
|
|
return
|
|
}
|
|
|
|
sort.Slice(objects, func(i, j int) bool {
|
|
if objects[i].IsDir == objects[j].IsDir {
|
|
return objects[i].FileName < objects[j].FileName
|
|
}
|
|
return objects[i].IsDir
|
|
})
|
|
|
|
tmpl, err := template.New("index").Funcs(template.FuncMap{
|
|
"formatTimestamp": formatTimestamp,
|
|
"formatSize": formatSize,
|
|
"trimPrefix": trimPrefix,
|
|
"urlencode": urlencode,
|
|
"parentDir": parentDir,
|
|
}).Parse(h.config.IndexPageTemplate())
|
|
if err != nil {
|
|
logAndSendBucketError(c, log, err)
|
|
return
|
|
}
|
|
if err = tmpl.Execute(c, &BrowsePageData{
|
|
BucketInfo: p.bucketInfo,
|
|
Prefix: p.prefix,
|
|
IsNative: p.isNative,
|
|
Objects: objects,
|
|
}); err != nil {
|
|
logAndSendBucketError(c, log, err)
|
|
return
|
|
}
|
|
}
|