package handler import ( "context" "html/template" "net/url" "sort" "strconv" "strings" "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/docker/go-units" "github.com/valyala/fasthttp" "go.uber.org/zap" ) const ( dateFormat = "02-01-2006 15:04" attrOID = "OID" attrCreated = "Created" attrFileName = "FileName" attrSize = "Size" ) type ( BrowsePageData struct { BucketInfo *data.BucketInfo Prefix string Objects []ResponseObject } ResponseObject struct { OID string Created string FileName string FilePath string Size string IsDir bool } ) func newListObjectsResponseS3(attrs map[string]string) ResponseObject { return ResponseObject{ OID: attrs[attrOID], Created: attrs[attrCreated], FileName: attrs[attrFileName], Size: attrs[attrSize], IsDir: attrs[attrOID] == "", } } func newListObjectsResponseNative(attrs map[string]string) ResponseObject { return ResponseObject{ OID: attrs[attrOID], Created: attrs[object.AttributeTimestamp] + "000", FileName: lastPathElement(attrs[object.AttributeFilePath]), FilePath: attrs[object.AttributeFilePath], Size: attrs[attrSize], IsDir: false, } } func getNextDir(filepath, prefix string) string { restPath := strings.Replace(filepath, prefix, "", 1) index := strings.Index(restPath, "/") if index == -1 { return "" } return restPath[:index] } func lastPathElement(path string) string { index := strings.LastIndex(path, "/") if index == len(path)-1 { index = strings.LastIndex(path[:index], "/") } return path[index+1:] } func parseTimestamp(tstamp string) (time.Time, error) { millis, err := strconv.ParseInt(tstamp, 10, 64) if err != nil { return time.Time{}, err } return time.UnixMilli(millis), nil } func formatTimestamp(strdate string) string { date, err := parseTimestamp(strdate) if err != nil || date.IsZero() { return "" } return date.Format(dateFormat) } func formatSize(strsize string) string { size, err := strconv.ParseFloat(strsize, 64) if err != nil { return "0B" } return units.HumanSize(size) } func parentDir(prefix string) string { index := strings.LastIndex(prefix, "/") if index == -1 { return prefix } return prefix[index:] } func trimPrefix(encPrefix string) string { prefix, err := url.PathUnescape(encPrefix) if err != nil { return "" } slashIndex := strings.LastIndex(prefix, "/") if slashIndex == -1 { return "" } return prefix[:slashIndex] } func urlencode(path string) string { var res strings.Builder prefixParts := strings.Split(path, "/") for _, prefixPart := range prefixParts { prefixPart = "/" + url.PathEscape(prefixPart) if prefixPart == "/." || prefixPart == "/.." { prefixPart = url.PathEscape(prefixPart) } res.WriteString(prefixPart) } return res.String() } func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) { nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) if err != nil { return nil, err } var objects = make([]ResponseObject, 0, len(nodes)) for _, node := range nodes { meta := node.GetMeta() if meta == nil { continue } var attrs = make(map[string]string, len(meta)) for _, m := range meta { attrs[m.GetKey()] = string(m.GetValue()) } obj := newListObjectsResponseS3(attrs) obj.FilePath = prefix + obj.FileName objects = append(objects, obj) } return objects, nil } func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) { basePath := strings.TrimRightFunc(prefix, func(r rune) bool { return r != '/' }) objectIDs, err := h.search(ctx, bucketInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { return nil, err } defer objectIDs.Close() objects, err := h.headDirObjects(ctx, bucketInfo.CID, objectIDs, basePath) if err != nil { return nil, err } return objects, nil } func (h *Handler) headDirObjects(ctx context.Context, cID cid.ID, objectIDs ResObjectSearch, basePath string) ([]ResponseObject, error) { const initialSliceCapacity = 100 var ( log = h.log.With( zap.String("cid", cID.EncodeToString()), zap.String("prefix", basePath), ) mu = sync.Mutex{} wg = sync.WaitGroup{} errChan = make(chan error) addr oid.Address objects = make([]ResponseObject, 0, initialSliceCapacity) dirs = sync.Map{} auth = PrmAuth{ BearerToken: bearerToken(ctx), } ) ctx, cancel := context.WithCancel(ctx) defer cancel() go func() { for err := range errChan { if err != nil { log.Error(logs.FailedToHeadObject, zap.Error(err)) } } }() addr.SetContainer(cID) err := objectIDs.Iterate(func(id oid.ID) bool { wg.Add(1) go func() { defer wg.Done() addr.SetObject(id) obj, err := h.frostfs.HeadObject(ctx, PrmObjectHead{ PrmAuth: auth, Address: addr, }) if err != nil { errChan <- err cancel() return } attrs := loadAttributes(obj.Attributes()) attrs[attrOID] = id.EncodeToString() attrs[attrSize] = strconv.FormatUint(obj.PayloadSize(), 10) dirname := getNextDir(attrs[object.AttributeFilePath], basePath) if dirname == "" { mu.Lock() objects = append(objects, newListObjectsResponseNative(attrs)) mu.Unlock() } else if _, ok := dirs.Load(dirname); !ok { mu.Lock() objects = append(objects, ResponseObject{ FileName: dirname, FilePath: basePath + dirname, IsDir: true, }) mu.Unlock() dirs.Store(dirname, true) } }() return false }) if err != nil { return nil, err } wg.Wait() close(errChan) return objects, nil } type browseParams struct { bucketInfo *data.BucketInfo template string listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) ([]ResponseObject, error) prefix string } func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) { log := h.log.With( zap.String("bucket", p.bucketInfo.Name), zap.String("container", p.bucketInfo.CID.EncodeToString()), zap.String("prefix", p.prefix), ) ctx := utils.GetContextFromRequest(c) objects, err := p.listObjects(ctx, p.bucketInfo, p.prefix) if err != nil { logAndSendBucketError(c, log, err) return } sort.Slice(objects, func(i, j int) bool { if objects[i].IsDir == objects[j].IsDir { return objects[i].FileName < objects[j].FileName } return objects[i].IsDir }) tmpl, err := template.New("index").Funcs(template.FuncMap{ "formatTimestamp": formatTimestamp, "formatSize": formatSize, "trimPrefix": trimPrefix, "urlencode": urlencode, "parentDir": parentDir, }).Parse(p.template) if err != nil { logAndSendBucketError(c, log, err) return } if err = tmpl.Execute(c, &BrowsePageData{ BucketInfo: p.bucketInfo, Prefix: p.prefix, Objects: objects, }); err != nil { logAndSendBucketError(c, log, err) return } }