diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index f8300ec..ac83621 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -218,20 +218,23 @@ func (a *app) loadIndexPageTemplate() { if !a.settings.IndexPageEnabled() { return } - reader, err := os.Open(a.cfg.GetString(cfgIndexPageTemplatePath)) + path := a.cfg.GetString(cfgIndexPageTemplatePath) + tmpl, err := a.readTemplate(path) if err != nil { a.settings.setIndexTemplate("") a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) - return + } else { + a.settings.setIndexTemplate(string(tmpl)) + a.log.Info(logs.SetCustomIndexPageTemplate, zap.String("path", path)) } - tmpl, err := io.ReadAll(reader) +} + +func (a *app) readTemplate(path string) ([]byte, error) { + reader, err := os.Open(path) if err != nil { - a.settings.setIndexTemplate("") - a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) - return + return nil, err } - a.settings.setIndexTemplate(string(tmpl)) - a.log.Info(logs.SetCustomIndexPageTemplate) + return io.ReadAll(reader) } func (s *appSettings) ClientCut() bool { diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index eab5b6b..d06c702 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -206,9 +206,6 @@ func settings() *viper.Viper { // pool: v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) - v.SetDefault(cfgIndexPageEnabled, false) - v.SetDefault(cfgIndexPageTemplatePath, "") - // frostfs: v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index e8d1f4b..3a78698 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -351,7 +351,12 @@ resolve_bucket: # `index_page` section -Parameters for index HTML-page output with S3-bucket or S3-subdir content for `Get object` request +Parameters for index HTML-page output. Activates if `GetObject` request returns `not found`. Two +index page modes available: + +* `s3` mode uses tree service for listing objects, +* `native` sends requests to nodes via native protocol. + If request pass S3-bucket name instead of CID, `s3` mode will be used, otherwise `native`. ```yaml index_page: diff --git a/internal/handler/browse.go b/internal/handler/browse.go index e84fb04..b16e5d4 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -1,15 +1,21 @@ package handler import ( + "context" "html/template" "net/url" "sort" "strconv" "strings" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/docker/go-units" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -25,19 +31,66 @@ const ( type ( BrowsePageData struct { - BucketName, - Prefix string - Objects []ResponseObject + BucketInfo *data.BucketInfo + Prefix string + Objects []ResponseObject + IsNative bool } ResponseObject struct { OID string Created string FileName string + FilePath string Size string IsDir bool } ) +func newListObjectsResponseS3(attrs map[string]string) ResponseObject { + return ResponseObject{ + OID: attrs[attrOID], + Created: attrs[attrCreated], + FileName: attrs[attrFileName], + Size: attrs[attrSize], + IsDir: attrs[attrOID] == "", + } +} + +func newListObjectsResponseNative(attrs map[string]string) ResponseObject { + filename := lastPathElement(attrs[object.AttributeFilePath]) + if filename == "" { + filename = attrs[attrFileName] + } + return ResponseObject{ + OID: attrs[attrOID], + Created: attrs[object.AttributeTimestamp] + "000", + FileName: filename, + FilePath: attrs[object.AttributeFilePath], + Size: attrs[attrSize], + IsDir: false, + } +} + +func getNextDir(filepath, prefix string) string { + restPath := strings.Replace(filepath, prefix, "", 1) + index := strings.Index(restPath, "/") + if index == -1 { + return "" + } + return restPath[:index] +} + +func lastPathElement(path string) string { + if path == "" { + return path + } + index := strings.LastIndex(path, "/") + if index == len(path)-1 { + index = strings.LastIndex(path[:index], "/") + } + return path[index+1:] +} + func parseTimestamp(tstamp string) (time.Time, error) { millis, err := strconv.ParseInt(tstamp, 10, 64) if err != nil { @@ -47,16 +100,6 @@ func parseTimestamp(tstamp string) (time.Time, error) { return time.UnixMilli(millis), nil } -func NewResponseObject(nodes map[string]string) ResponseObject { - return ResponseObject{ - OID: nodes[attrOID], - Created: nodes[attrCreated], - FileName: nodes[attrFileName], - Size: nodes[attrSize], - IsDir: nodes[attrOID] == "", - } -} - func formatTimestamp(strdate string) string { date, err := parseTimestamp(strdate) if err != nil || date.IsZero() { @@ -94,12 +137,9 @@ func trimPrefix(encPrefix string) string { return prefix[:slashIndex] } -func urlencode(prefix, filename string) string { +func urlencode(path string) string { var res strings.Builder - path := filename - if prefix != "" { - path = strings.Join([]string{prefix, filename}, "/") - } + prefixParts := strings.Split(path, "/") for _, prefixPart := range prefixParts { prefixPart = "/" + url.PathEscape(prefixPart) @@ -112,28 +152,185 @@ func urlencode(prefix, filename string) string { return res.String() } -func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) { - log := h.log.With(zap.String("bucket", bucketInfo.Name)) +func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) { + nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) + if err != nil { + return nil, err + } + + var objects = make([]ResponseObject, 0, len(nodes)) + for _, node := range nodes { + meta := node.GetMeta() + if meta == nil { + continue + } + var attrs = make(map[string]string, len(meta)) + for _, m := range meta { + attrs[m.GetKey()] = string(m.GetValue()) + } + obj := newListObjectsResponseS3(attrs) + obj.FilePath = prefix + obj.FileName + objects = append(objects, obj) + } + + return objects, nil +} + +type headDirParams struct { + cnrID cid.ID + objectIDs ResObjectSearch + basePath string + objCh chan<- ResponseObject + errCh chan<- error +} + +func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]ResponseObject, error) { + const initialSliceCapacity = 100 + + log := h.log.With( + zap.String("cid", bucketInfo.CID.EncodeToString()), + zap.String("prefix", prefix), + ) + basePath := strings.TrimRightFunc(prefix, func(r rune) bool { + return r != '/' + }) + filters := []object.SearchMatchType{object.MatchCommonPrefix} + if basePath == "" { + filters = append(filters, object.MatchNotPresent) + } + objCh := make(chan ResponseObject) + errCh := make(chan error) + done := make(chan struct{}) + objects := make([]ResponseObject, 0, initialSliceCapacity) + + go func() { + for err := range errCh { + if err != nil { + log.Error(logs.FailedToHeadObject, zap.Error(err)) + } + } + done <- struct{}{} + }() + go func() { + for obj := range objCh { + objects = append(objects, obj) + } + done <- struct{}{} + }() + + wg := sync.WaitGroup{} + for _, filter := range filters { + wg.Add(1) + go func(filter object.SearchMatchType) { + defer wg.Done() + objectIDs, err := h.search(ctx, bucketInfo.CID, object.AttributeFilePath, prefix, filter) + if err != nil { + errCh <- err + return + } + defer objectIDs.Close() + h.headDirObjects(ctx, headDirParams{ + cnrID: bucketInfo.CID, + objectIDs: objectIDs, + basePath: basePath, + objCh: objCh, + errCh: errCh, + }) + }(filter) + } + + wg.Wait() + close(errCh) + close(objCh) + <-done + <-done + + return objects, nil +} + +func (h *Handler) headDirObjects(ctx context.Context, p headDirParams) { + wg := sync.WaitGroup{} + dirs := sync.Map{} + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + err := p.objectIDs.Iterate(func(id oid.ID) bool { + wg.Add(1) + go func(id oid.ID) { + defer wg.Done() + h.headDirObject(ctx, id, p, &dirs, cancel) + }(id) + + select { + case <-ctx.Done(): + return true + default: + return false + } + }) + wg.Wait() + + if err != nil { + p.errCh <- err + return + } +} + +func (h *Handler) headDirObject(ctx context.Context, id oid.ID, p headDirParams, dirs *sync.Map, cancel context.CancelFunc) { + addr := newAddress(p.cnrID, id) + obj, err := h.frostfs.HeadObject(ctx, PrmObjectHead{ + PrmAuth: PrmAuth{BearerToken: bearerToken(ctx)}, + Address: addr, + }) + if err != nil { + p.errCh <- err + cancel() + return + } + + attrs := loadAttributes(obj.Attributes()) + attrs[attrOID] = id.EncodeToString() + attrs[attrSize] = strconv.FormatUint(obj.PayloadSize(), 10) + + dirname := getNextDir(attrs[object.AttributeFilePath], p.basePath) + if dirname == "" { + p.objCh <- newListObjectsResponseNative(attrs) + } else if _, ok := dirs.Load(dirname); !ok { + p.objCh <- ResponseObject{ + FileName: dirname, + FilePath: p.basePath + dirname, + IsDir: true, + } + dirs.Store(dirname, true) + } +} + +type browseParams struct { + bucketInfo *data.BucketInfo + prefix string + isNative bool + listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) ([]ResponseObject, error) +} + +func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) { + log := h.log.With( + zap.String("bucket", p.bucketInfo.Name), + zap.String("container", p.bucketInfo.CID.EncodeToString()), + zap.String("prefix", p.prefix), + ) ctx := utils.GetContextFromRequest(c) - nodes, err := h.listObjects(ctx, bucketInfo, prefix) + objects, err := p.listObjects(ctx, p.bucketInfo, p.prefix) if err != nil { logAndSendBucketError(c, log, err) return } - respObjects := make([]ResponseObject, len(nodes)) - - for i, node := range nodes { - respObjects[i] = NewResponseObject(node) - } - - sort.Slice(respObjects, func(i, j int) bool { - if respObjects[i].IsDir == respObjects[j].IsDir { - return respObjects[i].FileName < respObjects[j].FileName + sort.Slice(objects, func(i, j int) bool { + if objects[i].IsDir == objects[j].IsDir { + return objects[i].FileName < objects[j].FileName } - return respObjects[i].IsDir + return objects[i].IsDir }) - indexTemplate := h.config.IndexPageTemplate() tmpl, err := template.New("index").Funcs(template.FuncMap{ "formatTimestamp": formatTimestamp, @@ -141,15 +338,16 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketI "trimPrefix": trimPrefix, "urlencode": urlencode, "parentDir": parentDir, - }).Parse(indexTemplate) + }).Parse(h.config.IndexPageTemplate()) if err != nil { logAndSendBucketError(c, log, err) return } if err = tmpl.Execute(c, &BrowsePageData{ - BucketName: bucketInfo.Name, - Prefix: prefix, - Objects: respObjects, + BucketInfo: p.bucketInfo, + Prefix: p.prefix, + IsNative: p.isNative, + Objects: objects, }); err != nil { logAndSendBucketError(c, log, err) return diff --git a/internal/handler/download.go b/internal/handler/download.go index 88109a6..a8c006f 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -23,10 +23,9 @@ import ( // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { - test, _ := c.UserValue("oid").(string) - var id oid.ID - err := id.DecodeString(test) - if err != nil { + cnrIDStr, _ := c.UserValue("cid").(string) + var cnrID cid.ID + if err := cnrID.DecodeString(cnrIDStr); err != nil { h.byObjectName(c, h.receiveFile) } else { h.byAddress(c, h.receiveFile) @@ -45,7 +44,7 @@ func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { h.byAttribute(c, h.receiveFile) } -func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) { +func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) { filters := object.NewSearchFilters() filters.AddRootFilter() filters.AddFilter(key, val, op) @@ -54,7 +53,7 @@ func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op PrmAuth: PrmAuth{ BearerToken: bearerToken(ctx), }, - Container: *cnrID, + Container: cnrID, Filters: filters, } @@ -101,7 +100,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { return } - resSearch, err := h.search(ctx, &bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) + resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index c680706..1ff9533 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -192,7 +192,7 @@ func New(params *AppParams, config Config, tree *tree.Tree) *Handler { func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { var ( idCnr, _ = c.UserValue("cid").(string) - idObj, _ = c.UserValue("oid").(string) + idObj, _ = url.PathUnescape(c.UserValue("oid").(string)) log = h.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) ) @@ -206,6 +206,18 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ objID := new(oid.ID) if err = objID.DecodeString(idObj); err != nil { + if h.config.IndexPageEnabled() { + var addr oid.Address + addr.SetContainer(bktInfo.CID) + c.SetStatusCode(fasthttp.StatusNotFound) + h.browseObjects(c, browseParams{ + bucketInfo: bktInfo, + prefix: idObj, + listObjects: h.getDirObjectsNative, + isNative: true, + }) + return + } log.Error(logs.WrongObjectID, zap.Error(err)) response.Error(c, "wrong object id", fasthttp.StatusBadRequest) return @@ -245,7 +257,12 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r if isDir(unescapedKey) || isContainerRoot(unescapedKey) { if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK { c.SetStatusCode(code) - h.browseObjects(c, bktInfo, unescapedKey) + h.browseObjects(c, browseParams{ + bucketInfo: bktInfo, + prefix: unescapedKey, + listObjects: h.getDirObjectsS3, + isNative: false, + }) return } } @@ -299,7 +316,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re return } - res, err := h.search(ctx, &bktInfo.CID, key, val, object.MatchStringEqual) + res, err := h.search(ctx, bktInfo.CID, key, val, object.MatchStringEqual) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -395,25 +412,3 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } - -func (h *Handler) listObjects(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]map[string]string, error) { - nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) - if err != nil { - return nil, err - } - - var objects = make([]map[string]string, 0, len(nodes)) - for _, node := range nodes { - meta := node.GetMeta() - if meta == nil { - continue - } - var obj = make(map[string]string, len(meta)) - for _, m := range meta { - obj[m.GetKey()] = string(m.GetValue()) - } - objects = append(objects, obj) - } - - return objects, nil -} diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 4fe9153..0065657 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -57,10 +57,10 @@ func (c *configMock) IndexPageEnabled() bool { return false } -func (c *configMock) IndexPageTemplatePath() string { +func (c *configMock) IndexPageTemplate() string { return "" } -func (c *configMock) IndexPageTemplate() string { +func (c *configMock) IndexPageNativeTemplate() string { return "" } diff --git a/internal/handler/utils.go b/internal/handler/utils.go index a944b67..42665a9 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -61,6 +62,14 @@ func checkErrorType(err error) int { } } +func loadAttributes(attrs []object.Attribute) map[string]string { + result := make(map[string]string) + for _, attr := range attrs { + result[attr.Key()] = attr.Value() + } + return result +} + func isValidToken(s string) bool { for _, c := range s { if c <= ' ' || c > 127 { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 96bdaa5..cf3e546 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -71,6 +71,7 @@ const ( AddedStoragePeer = "added storage peer" // Info in ../../settings.go CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go + FailedToHeadObject = "failed to HEAD object" // Error in ../handler/handler.go InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go diff --git a/internal/templates/index.gotmpl b/internal/templates/index.gotmpl index ea66a62..27d5cef 100644 --- a/internal/templates/index.gotmpl +++ b/internal/templates/index.gotmpl @@ -1,10 +1,15 @@ -{{$bucketName := .BucketName}} +{{$isNative := .IsNative}} +{{$container := .BucketInfo.Name}} +{{if $isNative}} + {{$container = .BucketInfo.CID}} +{{end}} {{ $prefix := trimPrefix .Prefix }} - Index of s3://{{$bucketName}}/{{if $prefix}}/{{$prefix}}/{{end}} + Index of {{if $isNative}}frostfs{{else}}s3{{end}}://{{$container}} + /{{if $prefix}}/{{$prefix}}/{{end}} -

Index of s3://{{$bucketName}}/{{if $prefix}}{{$prefix}}/{{end}}

+

Index of {{if $isNative}}frostfs{{else}}s3{{end}}://{{$container}} + /{{if $prefix}}{{$prefix}}/{{end}}

+ @@ -42,47 +53,77 @@ {{if $trimmedPrefix }} + {{else}} + {{end}} {{range .Objects}} - - - - - - + {{if $isNative}} + + + + + + + + {{else}} + + + + + + + {{end}} {{end}}
FilenameOID Size Created Download
- ⮐.. + ⮐..
- ⮐.. + ⮐..
- {{if .IsDir}} - 🗀 - - {{.FileName}}/ - - {{else}} - 🗎 - - {{.FileName}} - - {{end}} - {{if not .IsDir}}{{ formatSize .Size }}{{end}}{{if not .IsDir}}{{ formatTimestamp .Created }}{{end}} - {{ if not .IsDir }} - - Link - - {{ end }} -
+ {{if .IsDir}} + 🗀 + + {{.FileName}}/ + + {{else}} + 🗎 + + {{.FileName}} + + {{end}} + {{.OID}}{{if .Size}}{{ formatSize .Size }}{{end}}{{if .Created}}{{ formatTimestamp .Created }}{{end}} + {{ if .OID }} + + Link + + {{ end }} +
+ {{if .IsDir}} + 🗀 + + {{.FileName}}/ + + {{else}} + 🗎 + + {{.FileName}} + + {{end}} + {{if not .IsDir}}{{ formatSize .Size }}{{end}}{{if not .IsDir}}{{ formatTimestamp .Created }}{{end}} + {{ if not .IsDir }} + + Link + + {{ end }} +