From cbaf9e61428772fd53effd52e5a7bf7ed0abaa43 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 3 Feb 2021 16:01:30 +0300 Subject: [PATCH] Fixes after review After discussion, we decided to simplify attribute translation for now. Full translation requires support for UTF-8 values encoded according to RFC 8187/7230, this can be clarified and implemented in further PRs. 1. Remove translation tables 2. Use simple translation rules only, for now ignoring UTF-8 keys and values - `X-Attribute-NEOFS-` prefixed headers considered well-known system attributes - System Attribute key is uppercased - System Attribute key's `-` translates to `_` - `X-Attribute-` prefixed headers considered regular object attributes - Normal attribute key is a result of http header prefix trimming - Value string should be left as is, for now ``` HTTP: X-Attribute-NEOFS-Expiration-Epoch: 123 X-Attribute-FileName: cat.jpg NeoFS: __NEOFS__EXPIRATION_EPOCH: "123" FileName: "cat.jpg" ``` Signed-off-by: Evgeniy Kulikov --- README.md | 9 ------ app.go | 4 --- filter.go | 88 ++++++++++++++++++----------------------------------- settings.go | 10 ------ upload.go | 2 +- 5 files changed, 30 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index d9f875b..f4b30d4 100644 --- a/README.md +++ b/README.md @@ -77,13 +77,4 @@ Peers preset: HTTP_GW_PEERS_[N]_ADDRESS = string HTTP_GW_PEERS_[N]_WEIGHT = 0..1 (float) - -Upload Header Table: - -HTTP_GW_UPLOADER_HEADER_[N]_KEY = string - HTTP Header attribute name prefixed with `X-Attribute-` -HTTP_GW_UPLOADER_HEADER_[N]_VAL = string - NeoFS Object attribute mapping - -# By default we had next headers: -- FileName - to set object filename attribute -- Timestamp - to set object timestamp attribute ``` \ No newline at end of file diff --git a/app.go b/app.go index 6d666eb..ca3d249 100644 --- a/app.go +++ b/app.go @@ -26,8 +26,6 @@ type ( cfg *viper.Viper key *ecdsa.PrivateKey - hdr HeaderFilter - wlog logger.Logger web *fasthttp.Server @@ -78,8 +76,6 @@ func newApp(ctx context.Context, opt ...Option) App { opt[i](a) } - a.hdr = newHeaderFilter(a.log, a.cfg) - a.enableDefaultTimestamp = a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) a.wlog = logger.GRPC(a.log) diff --git a/filter.go b/filter.go index 7e13766..7d8b24d 100644 --- a/filter.go +++ b/filter.go @@ -2,63 +2,33 @@ package main import ( "bytes" - "strconv" - "strings" - "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" ) -type ( - HeaderFilter interface { - Filter(header *fasthttp.RequestHeader) map[string]string - } +const ( + userAttributeHeaderPrefix = "X-Attribute-" + neofsAttributeHeaderPrefix = "NEOFS-" - headerFilter struct { - logger *zap.Logger - mapping map[string]string - } + systemAttributePrefix = "__NEOFS__" ) -const userAttributeHeaderPrefix = "X-Attribute-" +func systemTranslator(key []byte) []byte { + // replace `NEOFS-` with `__NEOFS__` + key = bytes.Replace(key, []byte(neofsAttributeHeaderPrefix), []byte(systemAttributePrefix), 1) -func newHeaderFilter(l *zap.Logger, v *viper.Viper) HeaderFilter { - filter := &headerFilter{ - logger: l, - mapping: make(map[string]string), - } + // replace `-` with `_` + key = bytes.ReplaceAll(key, []byte("-"), []byte("_")) - for i := 0; ; i++ { - index := strconv.Itoa(i) - key := strings.Join([]string{cfgUploaderHeader, index, cfgUploaderHeaderKey}, ".") - rep := strings.Join([]string{cfgUploaderHeader, index, cfgUploaderHeaderVal}, ".") - - keyValue := v.GetString(key) - repValue := v.GetString(rep) - - if keyValue == "" || repValue == "" { - break - } - - filter.mapping[keyValue] = repValue - - l.Debug("load upload header table value", - zap.String("key", keyValue), - zap.String("val", repValue)) - } - - // Default values - filter.mapping[object.AttributeFileName] = object.AttributeFileName - filter.mapping[object.AttributeTimestamp] = object.AttributeTimestamp - - return filter + // replace with uppercase + return bytes.ToUpper(key) } -func (h *headerFilter) Filter(header *fasthttp.RequestHeader) map[string]string { +func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]string { result := make(map[string]string) prefix := []byte(userAttributeHeaderPrefix) + system := []byte(neofsAttributeHeaderPrefix) header.VisitAll(func(key, val []byte) { // checks that key and val not empty @@ -71,27 +41,27 @@ func (h *headerFilter) Filter(header *fasthttp.RequestHeader) map[string]string return } - // checks that after removing attribute prefix we had not empty key - if key = bytes.TrimPrefix(key, prefix); len(key) == 0 { + // removing attribute prefix + key = bytes.TrimPrefix(key, prefix) + + // checks that it's a system NeoFS header + if bytes.HasPrefix(key, system) { + key = systemTranslator(key) + } + + // checks that attribute key not empty + if len(key) == 0 { return } - // checks mapping table and if we found record store it - // at resulting hashmap - if name, ok := h.mapping[string(key)]; ok { - result[name] = string(val) + // make string representation of key / val + k, v := string(key), string(val) - h.logger.Debug("add attribute to result object", - zap.String("key", name), - zap.String("val", string(val))) + result[k] = v - return - } - - // otherwise inform that attribute will be ignored - h.logger.Debug("ignore attribute", - zap.String("key", string(key)), - zap.String("val", string(val))) + l.Debug("add attribute to result object", + zap.String("key", k), + zap.String("val", v)) }) return result diff --git a/settings.go b/settings.go index 1775569..02fcde8 100644 --- a/settings.go +++ b/settings.go @@ -54,9 +54,6 @@ const ( cfgLoggerSamplingThereafter = "logger.sampling.thereafter" // Uploader Header - cfgUploaderHeader = "uploader_header" - cfgUploaderHeaderKey = "key" - cfgUploaderHeaderVal = "val" cfgUploaderHeaderEnableDefaultTimestamp = "upload_header.use_default_timestamp" // Peers @@ -187,13 +184,6 @@ func settings() *viper.Viper { fmt.Printf("%s_%s_[N]_ADDRESS = string\n", Prefix, strings.ToUpper(cfgPeers)) fmt.Printf("%s_%s_[N]_WEIGHT = 0..1 (float)\n", Prefix, strings.ToUpper(cfgPeers)) - fmt.Println() - fmt.Println("Upload Header Table:") - fmt.Println() - - fmt.Printf("%s_%s_[N]_%s = string\n", Prefix, strings.ToUpper(cfgUploaderHeader), strings.ToUpper(cfgUploaderHeaderKey)) - fmt.Printf("%s_%s_[N]_%s = string\n", Prefix, strings.ToUpper(cfgUploaderHeader), strings.ToUpper(cfgUploaderHeaderVal)) - os.Exit(0) case version != nil && *version: fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) diff --git a/upload.go b/upload.go index 12111b0..767ee2e 100644 --- a/upload.go +++ b/upload.go @@ -103,7 +103,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { } } - filtered := a.hdr.Filter(&c.Request.Header) + filtered := filterHeaders(a.log, &c.Request.Header) attributes := make([]*object.Attribute, 0, len(filtered)) // prepares attributes from filtered headers