316 lines
7.2 KiB
Go
316 lines
7.2 KiB
Go
package handler
|
|
|
|
import (
|
|
"context"
|
|
"html/template"
|
|
"net/url"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
"github.com/docker/go-units"
|
|
"github.com/valyala/fasthttp"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
const (
|
|
dateFormat = "02-01-2006 15:04"
|
|
attrOID = "OID"
|
|
attrCreated = "Created"
|
|
attrFileName = "FileName"
|
|
attrSize = "Size"
|
|
)
|
|
|
|
type (
|
|
BrowsePageData struct {
|
|
BucketInfo *data.BucketInfo
|
|
Prefix string
|
|
Objects []ResponseObject
|
|
}
|
|
ResponseObject struct {
|
|
OID string
|
|
Created string
|
|
FileName string
|
|
FilePath string
|
|
Size string
|
|
IsDir bool
|
|
}
|
|
)
|
|
|
|
func newListObjectsResponseS3(attrs map[string]string) ResponseObject {
|
|
return ResponseObject{
|
|
OID: attrs[attrOID],
|
|
Created: attrs[attrCreated],
|
|
FileName: attrs[attrFileName],
|
|
Size: attrs[attrSize],
|
|
IsDir: attrs[attrOID] == "",
|
|
}
|
|
}
|
|
|
|
func newListObjectsResponseNative(attrs map[string]string) ResponseObject {
|
|
return ResponseObject{
|
|
OID: attrs[attrOID],
|
|
Created: attrs[object.AttributeTimestamp] + "000",
|
|
FileName: lastPathElement(attrs[object.AttributeFilePath]),
|
|
FilePath: attrs[object.AttributeFilePath],
|
|
Size: attrs[attrSize],
|
|
IsDir: false,
|
|
}
|
|
}
|
|
|
|
func getNextDir(filepath, prefix string) string {
|
|
restPath := strings.Replace(filepath, prefix, "", 1)
|
|
index := strings.Index(restPath, "/")
|
|
if index == -1 {
|
|
return ""
|
|
}
|
|
return restPath[:index]
|
|
}
|
|
|
|
func lastPathElement(path string) string {
|
|
index := strings.LastIndex(path, "/")
|
|
if index == len(path)-1 {
|
|
index = strings.LastIndex(path[:index], "/")
|
|
}
|
|
return path[index+1:]
|
|
}
|
|
|
|
func parseTimestamp(tstamp string) (time.Time, error) {
|
|
millis, err := strconv.ParseInt(tstamp, 10, 64)
|
|
if err != nil {
|
|
return time.Time{}, err
|
|
}
|
|
|
|
return time.UnixMilli(millis), nil
|
|
}
|
|
|
|
func formatTimestamp(strdate string) string {
|
|
date, err := parseTimestamp(strdate)
|
|
if err != nil || date.IsZero() {
|
|
return ""
|
|
}
|
|
|
|
return date.Format(dateFormat)
|
|
}
|
|
|
|
func formatSize(strsize string) string {
|
|
size, err := strconv.ParseFloat(strsize, 64)
|
|
if err != nil {
|
|
return "0B"
|
|
}
|
|
return units.HumanSize(size)
|
|
}
|
|
|
|
func parentDir(prefix string) string {
|
|
index := strings.LastIndex(prefix, "/")
|
|
if index == -1 {
|
|
return prefix
|
|
}
|
|
return prefix[index:]
|
|
}
|
|
|
|
func trimPrefix(encPrefix string) string {
|
|
prefix, err := url.PathUnescape(encPrefix)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
slashIndex := strings.LastIndex(prefix, "/")
|
|
if slashIndex == -1 {
|
|
return ""
|
|
}
|
|
return prefix[:slashIndex]
|
|
}
|
|
|
|
func urlencode(path string) string {
|
|
var res strings.Builder
|
|
|
|
prefixParts := strings.Split(path, "/")
|
|
for _, prefixPart := range prefixParts {
|
|
prefixPart = "/" + url.PathEscape(prefixPart)
|
|
if prefixPart == "/." || prefixPart == "/.." {
|
|
prefixPart = url.PathEscape(prefixPart)
|
|
}
|
|
res.WriteString(prefixPart)
|
|
}
|
|
|
|
return res.String()
|
|
}
|
|
|
|
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) {
|
|
nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var objects = make([]ResponseObject, 0, len(nodes))
|
|
for _, node := range nodes {
|
|
meta := node.GetMeta()
|
|
if meta == nil {
|
|
continue
|
|
}
|
|
var attrs = make(map[string]string, len(meta))
|
|
for _, m := range meta {
|
|
attrs[m.GetKey()] = string(m.GetValue())
|
|
}
|
|
obj := newListObjectsResponseS3(attrs)
|
|
obj.FilePath = prefix + obj.FileName
|
|
objects = append(objects, obj)
|
|
}
|
|
|
|
return objects, nil
|
|
}
|
|
|
|
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) {
|
|
basePath := strings.TrimRightFunc(prefix, func(r rune) bool {
|
|
return r != '/'
|
|
})
|
|
objectIDs, err := h.search(ctx, bucketInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer objectIDs.Close()
|
|
|
|
objects, err := h.headDirObjects(ctx, bucketInfo.CID, objectIDs, basePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return objects, nil
|
|
}
|
|
|
|
func (h *Handler) headDirObjects(ctx context.Context, cID cid.ID, objectIDs ResObjectSearch, basePath string) ([]ResponseObject, error) {
|
|
const initialSliceCapacity = 100
|
|
|
|
var (
|
|
log = h.log.With(
|
|
zap.String("cid", cID.EncodeToString()),
|
|
zap.String("prefix", basePath),
|
|
)
|
|
mu = sync.Mutex{}
|
|
wg = sync.WaitGroup{}
|
|
errChan = make(chan error)
|
|
|
|
addr oid.Address
|
|
objects = make([]ResponseObject, 0, initialSliceCapacity)
|
|
dirs = sync.Map{}
|
|
auth = PrmAuth{
|
|
BearerToken: bearerToken(ctx),
|
|
}
|
|
)
|
|
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
|
|
go func() {
|
|
for err := range errChan {
|
|
if err != nil {
|
|
log.Error(logs.FailedToHeadObject, zap.Error(err))
|
|
}
|
|
}
|
|
}()
|
|
|
|
addr.SetContainer(cID)
|
|
err := objectIDs.Iterate(func(id oid.ID) bool {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
|
|
addr.SetObject(id)
|
|
obj, err := h.frostfs.HeadObject(ctx, PrmObjectHead{
|
|
PrmAuth: auth,
|
|
Address: addr,
|
|
})
|
|
if err != nil {
|
|
errChan <- err
|
|
cancel()
|
|
return
|
|
}
|
|
|
|
attrs := loadAttributes(obj.Attributes())
|
|
attrs[attrOID] = id.EncodeToString()
|
|
attrs[attrSize] = strconv.FormatUint(obj.PayloadSize(), 10)
|
|
|
|
dirname := getNextDir(attrs[object.AttributeFilePath], basePath)
|
|
if dirname == "" {
|
|
mu.Lock()
|
|
objects = append(objects, newListObjectsResponseNative(attrs))
|
|
mu.Unlock()
|
|
} else if _, ok := dirs.Load(dirname); !ok {
|
|
mu.Lock()
|
|
objects = append(objects, ResponseObject{
|
|
FileName: dirname,
|
|
FilePath: basePath + dirname,
|
|
IsDir: true,
|
|
})
|
|
mu.Unlock()
|
|
dirs.Store(dirname, true)
|
|
}
|
|
}()
|
|
|
|
return false
|
|
})
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
wg.Wait()
|
|
close(errChan)
|
|
|
|
return objects, nil
|
|
}
|
|
|
|
type browseParams struct {
|
|
bucketInfo *data.BucketInfo
|
|
template string
|
|
listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) ([]ResponseObject, error)
|
|
prefix string
|
|
}
|
|
|
|
func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) {
|
|
log := h.log.With(
|
|
zap.String("bucket", p.bucketInfo.Name),
|
|
zap.String("container", p.bucketInfo.CID.EncodeToString()),
|
|
zap.String("prefix", p.prefix),
|
|
)
|
|
ctx := utils.GetContextFromRequest(c)
|
|
objects, err := p.listObjects(ctx, p.bucketInfo, p.prefix)
|
|
if err != nil {
|
|
logAndSendBucketError(c, log, err)
|
|
return
|
|
}
|
|
|
|
sort.Slice(objects, func(i, j int) bool {
|
|
if objects[i].IsDir == objects[j].IsDir {
|
|
return objects[i].FileName < objects[j].FileName
|
|
}
|
|
return objects[i].IsDir
|
|
})
|
|
|
|
tmpl, err := template.New("index").Funcs(template.FuncMap{
|
|
"formatTimestamp": formatTimestamp,
|
|
"formatSize": formatSize,
|
|
"trimPrefix": trimPrefix,
|
|
"urlencode": urlencode,
|
|
"parentDir": parentDir,
|
|
}).Parse(p.template)
|
|
if err != nil {
|
|
logAndSendBucketError(c, log, err)
|
|
return
|
|
}
|
|
if err = tmpl.Execute(c, &BrowsePageData{
|
|
BucketInfo: p.bucketInfo,
|
|
Prefix: p.prefix,
|
|
Objects: objects,
|
|
}); err != nil {
|
|
logAndSendBucketError(c, log, err)
|
|
return
|
|
}
|
|
}
|