From a8ec09e76a1770186b3fd47cf5b67189d1baf251 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 16 Mar 2023 11:40:08 +0300 Subject: [PATCH] [#22] Update system attributes prefix Signed-off-by: Denis Kirillov --- README.md | 22 ++-- docs/api.md | 108 ++++++++-------- downloader/download.go | 38 +----- downloader/download_test.go | 23 ---- downloader/head.go | 7 +- go.mod | 4 +- go.sum | 8 +- integration_test.go | 7 +- uploader/filter.go | 91 +------------ uploader/filter_test.go | 177 ++----------------------- uploader/upload.go | 62 ++------- utils/attributes.go | 250 +++++++++++++++++++++++++++++++++++- utils/attributes_test.go | 187 +++++++++++++++++++++++++++ utils/util.go | 26 ++++ 14 files changed, 562 insertions(+), 448 deletions(-) delete mode 100644 downloader/download_test.go create mode 100644 utils/attributes_test.go diff --git a/README.md b/README.md index de81301..3acf639 100644 --- a/README.md +++ b/README.md @@ -424,12 +424,12 @@ You can also add some attributes to your file using the following rules: "X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo: 100500" header to your request the resulting object will get "Ololo: 100500" attribute - * "X-Attribute-NEOFS-*" headers are special - (`-NEOFS-` part can also be `-neofs-` or`-Neofs-`), they're used to set internal - NeoFS attributes starting with `__NEOFS__` prefix, for these attributes all + * "X-Attribute-SYSTEM-*" headers are special + (`-SYSTEM-` part can also be `-system-` or`-System-` (and even legacy `-Neofs-` for some next releases)), they're used to set internal + FrostFS attributes starting with `__SYSTEM__` prefix, for these attributes all dashes get converted to underscores and all letters are capitalized. For - example, you can use "X-Attribute-NEOFS-Expiration-Epoch" header to set - `__NEOFS__EXPIRATION_EPOCH` attribute + example, you can use "X-Attribute-SYSTEM-Expiration-Epoch" header to set + `__SYSTEM__EXPIRATION_EPOCH` attribute * `FileName` attribute is set from multipart's `filename` if not set explicitly via `X-Attribute-FileName` header * `Timestamp` attribute can be set using gateway local time if using @@ -439,13 +439,13 @@ You can also add some attributes to your file using the following rules: --- **NOTE** -There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority): -1. `X-Attribute-Neofs-Expiration-Epoch: 100` -2. `X-Attribute-Neofs-Expiration-Duration: 24h30m` -3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797` -4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z` +There are some reserved headers type of `X-Attribute-SYSTEM-*` (headers are arranged in descending order of priority): +1. `X-Attribute-System-Expiration-Epoch: 100` +2. `X-Attribute-System-Expiration-Duration: 24h30m` +3. `X-Attribute-System-Expiration-Timestamp: 1637574797` +4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z` -which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way. +which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way. --- diff --git a/docs/api.md b/docs/api.md index 6a19465..78df766 100644 --- a/docs/api.md +++ b/docs/api.md @@ -56,21 +56,21 @@ Upload file as object with attributes to FrostFS. ###### Headers -| Header | Description | -|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------| -| Common headers | See [bearer token](#bearer-token). | -| `X-Attribute-Neofs-*` | Used to set system NeoFS object attributes
(e.g. use "X-Attribute-Neofs-Expiration-Epoch" to set `__NEOFS__EXPIRATION_EPOCH` attribute). | -| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | -| `Date` | This header is used to calculate the right `__NEOFS__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | +| Header | Description | +|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| +| Common headers | See [bearer token](#bearer-token). | +| `X-Attribute-System-*` | Used to set system FrostFS object attributes
(e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). | +| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | +| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | -There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority): +There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority): -1. `X-Attribute-Neofs-Expiration-Epoch: 100` -2. `X-Attribute-Neofs-Expiration-Duration: 24h30m` -3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797` -4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z` +1. `X-Attribute-System-Expiration-Epoch: 100` +2. `X-Attribute-System-Expiration-Duration: 24h30m` +3. `X-Attribute-System-Expiration-Timestamp: 1637574797` +4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z` -which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way. +which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way. If you don't specify the `X-Attribute-Timestamp` header the `Timestamp` attribute can be set anyway (see http-gw [configuration](gate-configuration.md#upload-header-section)). @@ -121,17 +121,17 @@ Get an object (payload and attributes) by an address. ###### Headers -| Header | Description | -|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------| -| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | -| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | -| `Content-Disposition` | Indicate how to browsers should treat file.
Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | -| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | -| `Content-Length` | Size of object payload. | -| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | -| `X-Owner-Id` | Base58 encoded owner ID. | -| `X-Container-Id` | Base58 encoded container ID. | -| `X-Object-Id` | Base58 encoded object ID. | +| Header | Description | +|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-System-*` | System FrostFS object attributes
(e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Disposition` | Indicate how to browsers should treat file.
Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | ###### Status codes @@ -157,16 +157,16 @@ Get an object attributes by an address. ###### Headers -| Header | Description | -|-----------------------|--------------------------------------------------------------------------------------------------------------------------| -| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | -| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | -| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | -| `Content-Length` | Size of object payload. | -| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | -| `X-Owner-Id` | Base58 encoded owner ID. | -| `X-Container-Id` | Base58 encoded container ID. | -| `X-Object-Id` | Base58 encoded object ID. | +| Header | Description | +|------------------------|------------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-System-*` | System FrostFS object attributes
(e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | ###### Status codes @@ -206,17 +206,17 @@ If more than one object is found, an arbitrary one will be returned. ###### Headers -| Header | Description | -|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------| -| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | -| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | -| `Content-Disposition` | Indicate how to browsers should treat file.
Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | -| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | -| `Content-Length` | Size of object payload. | -| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | -| `X-Owner-Id` | Base58 encoded owner ID. | -| `X-Container-Id` | Base58 encoded container ID. | -| `X-Object-Id` | Base58 encoded object ID. | +| Header | Description | +|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-System-*` | System FrostFS object attributes
(e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Disposition` | Indicate how to browsers should treat file.
Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | ###### Status codes @@ -243,16 +243,16 @@ If more than one object is found, an arbitrary one will be used to get attribute ###### Headers -| Header | Description | -|-----------------------|--------------------------------------------------------------------------------------------------------------------------| -| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | -| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | -| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | -| `Content-Length` | Size of object payload. | -| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | -| `X-Owner-Id` | Base58 encoded owner ID. | -| `X-Container-Id` | Base58 encoded container ID. | -| `X-Object-Id` | Base58 encoded object ID. | +| Header | Description | +|------------------------|------------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-System-*` | System FrostFS object attributes
(e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | ###### Status codes diff --git a/downloader/download.go b/downloader/download.go index d8f0bc2..6d24daa 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -14,8 +14,6 @@ import ( "strconv" "strings" "time" - "unicode" - "unicode/utf8" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" @@ -131,9 +129,9 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) { if !isValidToken(key) || !isValidValue(val) { continue } - if strings.HasPrefix(key, utils.SystemAttributePrefix) { - key = systemBackwardTranslator(key) - } + + key = utils.BackwardTransformIfSystem(key) + r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) switch key { case object.AttributeFileName: @@ -187,36 +185,6 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) { r.Response.SetBodyStream(rObj.Payload, int(payloadSize)) } -// systemBackwardTranslator is used to convert headers looking like '__NEOFS__ATTR_NAME' to 'Neofs-Attr-Name'. -func systemBackwardTranslator(key string) string { - // trim specified prefix '__NEOFS__' - key = strings.TrimPrefix(key, utils.SystemAttributePrefix) - - var res strings.Builder - res.WriteString("Neofs-") - - strs := strings.Split(key, "_") - for i, s := range strs { - s = title(strings.ToLower(s)) - res.WriteString(s) - if i != len(strs)-1 { - res.WriteString("-") - } - } - - return res.String() -} - -func title(str string) string { - if str == "" { - return "" - } - - r, size := utf8.DecodeRuneInString(str) - r0 := unicode.ToTitle(r) - return string(r0) + str[size:] -} - func bearerToken(ctx context.Context) *bearer.Token { if tkn, err := tokens.LoadBearerToken(ctx); err == nil { return tkn diff --git a/downloader/download_test.go b/downloader/download_test.go deleted file mode 100644 index a47d12a..0000000 --- a/downloader/download_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package downloader - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSystemBackwardTranslator(t *testing.T) { - input := []string{ - "__NEOFS__EXPIRATION_EPOCH", - "__NEOFS__RANDOM_ATTR", - } - expected := []string{ - "Neofs-Expiration-Epoch", - "Neofs-Random-Attr", - } - - for i, str := range input { - res := systemBackwardTranslator(str) - require.Equal(t, expected[i], res) - } -} diff --git a/downloader/head.go b/downloader/head.go index 36f8b60..4615103 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -4,7 +4,6 @@ import ( "io" "net/http" "strconv" - "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" @@ -56,9 +55,9 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { if !isValidToken(key) || !isValidValue(val) { continue } - if strings.HasPrefix(key, utils.SystemAttributePrefix) { - key = systemBackwardTranslator(key) - } + + key = utils.BackwardTransformIfSystem(key) + r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) switch key { case object.AttributeTimestamp: diff --git a/go.mod b/go.mod index 4b7319e..0ce5791 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.18 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.0 github.com/prometheus/client_golang v1.13.0 diff --git a/go.sum b/go.sum index 585439e..7d4f24e 100644 --- a/go.sum +++ b/go.sum @@ -37,14 +37,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51 h1:l4+K1hN+NuWNtlZZoV8yRRP3Uu7PifL05ukEqKcb0Ks= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51/go.mod h1:n0DxKYulu2Ar73R6OcNF34LiL/Xa+iDR7GZuaOChbLE= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 h1:lxe0DtZq/uFZVZu9apx6OcIXCJskQBMd/GVeYGKA3wA= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703/go.mod h1:gRd5iE5A84viily6AcNBsSlTx2XgoWrwRDz7z0MayDQ= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599 h1:mzGX2RX8R8H/tUqrUu1TcYk4QRDBcBIWGYscPncfLOQ= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599/go.mod h1:z7zcpGY+puI5puyy5oyFbf20vWp84WtslCxcr6/kv5c= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 h1:TUcJ5A0C1gWi3bAhw4b+V+iVM3E9mbBOdJIWWkAPNxo= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85/go.mod h1:23fUGlEv/ImaOi3vck6vZj0v0b4hteOhLLPnVWHSQeA= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/integration_test.go b/integration_test.go index 0587273..40d908b 100644 --- a/integration_test.go +++ b/integration_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + containerv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -407,7 +408,11 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o if version >= versionWithNativeNames { var domain container.Domain domain.SetName(testContainerName) - container.WriteDomain(&cnr, domain) + + // currently node in aio image knows nothing about new sys attributes + // todo (@dkirillov): #2 use frostfs aio images that supports new attributes + cnr.SetAttribute(containerv2.SysAttributeNameNeoFS, domain.Name()) + cnr.SetAttribute(containerv2.SysAttributeZoneNeoFS, domain.Zone()) } var waitPrm pool.WaitParams diff --git a/uploader/filter.go b/uploader/filter.go index acab646..35de625 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -3,29 +3,12 @@ package uploader import ( "bytes" "fmt" - "math" - "strconv" - "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "github.com/valyala/fasthttp" "go.uber.org/zap" ) -var frostfsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")} - -func systemTranslator(key, prefix []byte) []byte { - // replace the specified prefix with `__NEOFS__` - key = bytes.Replace(key, prefix, []byte(utils.SystemAttributePrefix), 1) - - // replace `-` with `_` - key = bytes.ReplaceAll(key, []byte("-"), []byte("_")) - - // replace with uppercase - return bytes.ToUpper(key) -} - func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]string, error) { var err error result := make(map[string]string) @@ -45,13 +28,7 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st // removing attribute prefix clearKey := bytes.TrimPrefix(key, prefix) - // checks that it's a system NeoFS header - for _, system := range frostfsAttributeHeaderPrefixes { - if bytes.HasPrefix(clearKey, system) { - clearKey = systemTranslator(clearKey, system) - break - } - } + clearKey = utils.TransformIfSystem(clearKey) // checks that the attribute key is not empty if len(clearKey) == 0 { @@ -77,69 +54,3 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st return result, err } - -func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations, now time.Time) error { - expirationInEpoch := headers[object.SysAttributeExpEpoch] - - if timeRFC3339, ok := headers[utils.ExpirationRFC3339Attr]; ok { - expTime, err := time.Parse(time.RFC3339, timeRFC3339) - if err != nil { - return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, utils.ExpirationRFC3339Attr) - } - - if expTime.Before(now) { - return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, utils.ExpirationRFC3339Attr) - } - updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) - delete(headers, utils.ExpirationRFC3339Attr) - } - - if timestamp, ok := headers[utils.ExpirationTimestampAttr]; ok { - value, err := strconv.ParseInt(timestamp, 10, 64) - if err != nil { - return fmt.Errorf("couldn't parse value %s of header %s", timestamp, utils.ExpirationTimestampAttr) - } - expTime := time.Unix(value, 0) - - if expTime.Before(now) { - return fmt.Errorf("value %s of header %s must be in the future", timestamp, utils.ExpirationTimestampAttr) - } - updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) - delete(headers, utils.ExpirationTimestampAttr) - } - - if duration, ok := headers[utils.ExpirationDurationAttr]; ok { - expDuration, err := time.ParseDuration(duration) - if err != nil { - return fmt.Errorf("couldn't parse value %s of header %s", duration, utils.ExpirationDurationAttr) - } - if expDuration <= 0 { - return fmt.Errorf("value %s of header %s must be positive", expDuration, utils.ExpirationDurationAttr) - } - updateExpirationHeader(headers, epochDurations, expDuration) - delete(headers, utils.ExpirationDurationAttr) - } - - if expirationInEpoch != "" { - headers[object.SysAttributeExpEpoch] = expirationInEpoch - } - - return nil -} - -func updateExpirationHeader(headers map[string]string, durations *epochDurations, expDuration time.Duration) { - epochDuration := uint64(durations.msPerBlock) * durations.blockPerEpoch - currentEpoch := durations.currentEpoch - numEpoch := uint64(expDuration.Milliseconds()) / epochDuration - - if uint64(expDuration.Milliseconds())%epochDuration != 0 { - numEpoch++ - } - - expirationEpoch := uint64(math.MaxUint64) - if numEpoch < math.MaxUint64-currentEpoch { - expirationEpoch = currentEpoch + numEpoch - } - - headers[object.SysAttributeExpEpoch] = strconv.FormatUint(expirationEpoch, 10) -} diff --git a/uploader/filter_test.go b/uploader/filter_test.go index 9cb58c1..1190001 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -1,13 +1,8 @@ package uploader import ( - "math" - "strconv" "testing" - "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -28,8 +23,8 @@ func TestFilter(t *testing.T) { t.Run("duplicate system keys error", func(t *testing.T) { req := &fasthttp.RequestHeader{} req.DisableNormalizing() - req.Add("X-Attribute-Neofs-DupKey", "first-value") - req.Add("X-Attribute-Neofs-DupKey", "second-value") + req.Add("X-Attribute-System-DupKey", "first-value") + req.Add("X-Attribute-System-DupKey", "second-value") _, err := filterHeaders(log, req) require.Error(t, err) }) @@ -37,16 +32,16 @@ func TestFilter(t *testing.T) { req := &fasthttp.RequestHeader{} req.DisableNormalizing() - req.Set("X-Attribute-Neofs-Expiration-Epoch1", "101") - req.Set("X-Attribute-NEOFS-Expiration-Epoch2", "102") - req.Set("X-Attribute-neofs-Expiration-Epoch3", "103") + req.Set("X-Attribute-System-Expiration-Epoch1", "101") + req.Set("X-Attribute-SYSTEM-Expiration-Epoch2", "102") + req.Set("X-Attribute-system-Expiration-Epoch3", "103") req.Set("X-Attribute-MyAttribute", "value") expected := map[string]string{ - "__NEOFS__EXPIRATION_EPOCH1": "101", - "MyAttribute": "value", - "__NEOFS__EXPIRATION_EPOCH3": "103", - "__NEOFS__EXPIRATION_EPOCH2": "102", + "__SYSTEM__EXPIRATION_EPOCH1": "101", + "MyAttribute": "value", + "__SYSTEM__EXPIRATION_EPOCH3": "103", + "__SYSTEM__EXPIRATION_EPOCH2": "102", } result, err := filterHeaders(log, req) @@ -54,157 +49,3 @@ func TestFilter(t *testing.T) { require.Equal(t, expected, result) } - -func TestPrepareExpirationHeader(t *testing.T) { - tomorrow := time.Now().Add(24 * time.Hour) - tomorrowUnix := tomorrow.Unix() - tomorrowUnixNano := tomorrow.UnixNano() - tomorrowUnixMilli := tomorrowUnixNano / 1e6 - - epoch := "100" - duration := "24h" - timestampSec := strconv.FormatInt(tomorrowUnix, 10) - timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10) - timestampNano := strconv.FormatInt(tomorrowUnixNano, 10) - - defaultDurations := &epochDurations{ - currentEpoch: 10, - msPerBlock: 1000, - blockPerEpoch: 101, - } - - msPerBlock := defaultDurations.blockPerEpoch * uint64(defaultDurations.msPerBlock) - epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock - if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 { - epochPerDay++ - } - - defaultExpEpoch := strconv.FormatUint(defaultDurations.currentEpoch+epochPerDay, 10) - - for _, tc := range []struct { - name string - headers map[string]string - durations *epochDurations - err bool - expected map[string]string - }{ - { - name: "valid epoch", - headers: map[string]string{object.SysAttributeExpEpoch: epoch}, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid epoch, valid duration", - headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - utils.ExpirationDurationAttr: duration, - }, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid epoch, valid rfc3339", - headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339), - }, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid epoch, valid timestamp sec", - headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - utils.ExpirationTimestampAttr: timestampSec, - }, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid epoch, valid timestamp milli", - headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - utils.ExpirationTimestampAttr: timestampMilli, - }, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid epoch, valid timestamp nano", - headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - utils.ExpirationTimestampAttr: timestampNano, - }, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid timestamp sec", - headers: map[string]string{utils.ExpirationTimestampAttr: timestampSec}, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, - }, - { - name: "valid duration", - headers: map[string]string{utils.ExpirationDurationAttr: duration}, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, - }, - { - name: "valid rfc3339", - headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)}, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, - }, - { - name: "valid max uint 64", - headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)}, - durations: &epochDurations{ - currentEpoch: math.MaxUint64 - 1, - msPerBlock: defaultDurations.msPerBlock, - blockPerEpoch: defaultDurations.blockPerEpoch, - }, - expected: map[string]string{object.SysAttributeExpEpoch: strconv.FormatUint(uint64(math.MaxUint64), 10)}, - }, - { - name: "invalid timestamp sec", - headers: map[string]string{utils.ExpirationTimestampAttr: "abc"}, - err: true, - }, - { - name: "invalid timestamp sec zero", - headers: map[string]string{utils.ExpirationTimestampAttr: "0"}, - err: true, - }, - { - name: "invalid duration", - headers: map[string]string{utils.ExpirationDurationAttr: "1d"}, - err: true, - }, - { - name: "invalid duration negative", - headers: map[string]string{utils.ExpirationDurationAttr: "-5h"}, - err: true, - }, - { - name: "invalid rfc3339", - headers: map[string]string{utils.ExpirationRFC3339Attr: "abc"}, - err: true, - }, - { - name: "invalid rfc3339 zero", - headers: map[string]string{utils.ExpirationRFC3339Attr: time.RFC3339}, - err: true, - }, - } { - t.Run(tc.name, func(t *testing.T) { - err := prepareExpirationHeader(tc.headers, tc.durations, time.Now()) - if tc.err { - require.Error(t, err) - } else { - require.NoError(t, err) - require.Equal(t, tc.expected, tc.headers) - } - }) - } -} diff --git a/uploader/upload.go b/uploader/upload.go index d455a13..4e21f08 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -3,7 +3,6 @@ package uploader import ( "context" "encoding/json" - "fmt" "io" "net/http" "strconv" @@ -38,12 +37,6 @@ type Uploader struct { containerResolver *resolver.ContainerResolver } -type epochDurations struct { - currentEpoch uint64 - msPerBlock int64 - blockPerEpoch uint64 -} - // Settings stores reloading parameters, so it has to provide atomic getters and setters. type Settings struct { defaultTimestamp atomic.Bool @@ -120,28 +113,20 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { response.Error(c, err.Error(), fasthttp.StatusBadRequest) return } - if needParseExpiration(filtered) { - epochDuration, err := getEpochDurations(c, u.pool) - if err != nil { - log.Error("could not get epoch durations from network info", zap.Error(err)) - response.Error(c, "could not get epoch durations from network info: "+err.Error(), fasthttp.StatusBadRequest) - return - } - now := time.Now() - if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { - if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { - log.Warn("could not parse client time", zap.String("Date header", string(rawHeader)), zap.Error(err)) - } else { - now = parsed - } + now := time.Now() + if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { + if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { + log.Warn("could not parse client time", zap.String("Date header", string(rawHeader)), zap.Error(err)) + } else { + now = parsed } + } - if err = prepareExpirationHeader(filtered, epochDuration, now); err != nil { - log.Error("could not parse expiration header", zap.Error(err)) - response.Error(c, "could not parse expiration header: "+err.Error(), fasthttp.StatusBadRequest) - return - } + if err = utils.PrepareExpirationHeader(c, u.pool, filtered, now); err != nil { + log.Error("could not prepare expiration header", zap.Error(err)) + response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) + return } attributes := make([]object.Attribute, 0, len(filtered)) @@ -246,28 +231,3 @@ func (pr *putResponse) encode(w io.Writer) error { enc.SetIndent("", "\t") return enc.Encode(pr) } - -func getEpochDurations(ctx context.Context, p *pool.Pool) (*epochDurations, error) { - networkInfo, err := p.NetworkInfo(ctx) - if err != nil { - return nil, err - } - - res := &epochDurations{ - currentEpoch: networkInfo.CurrentEpoch(), - msPerBlock: networkInfo.MsPerBlock(), - blockPerEpoch: networkInfo.EpochDuration(), - } - - if res.blockPerEpoch == 0 { - return nil, fmt.Errorf("EpochDuration is empty") - } - return res, nil -} - -func needParseExpiration(headers map[string]string) bool { - _, ok1 := headers[utils.ExpirationDurationAttr] - _, ok2 := headers[utils.ExpirationRFC3339Attr] - _, ok3 := headers[utils.ExpirationTimestampAttr] - return ok1 || ok2 || ok3 -} diff --git a/utils/attributes.go b/utils/attributes.go index 814d7b1..cfa3e3a 100644 --- a/utils/attributes.go +++ b/utils/attributes.go @@ -1,10 +1,250 @@ package utils +import ( + "bytes" + "context" + "errors" + "fmt" + "math" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" +) + const ( UserAttributeHeaderPrefix = "X-Attribute-" - SystemAttributePrefix = "__NEOFS__" - - ExpirationDurationAttr = SystemAttributePrefix + "EXPIRATION_DURATION" - ExpirationTimestampAttr = SystemAttributePrefix + "EXPIRATION_TIMESTAMP" - ExpirationRFC3339Attr = SystemAttributePrefix + "EXPIRATION_RFC3339" ) + +const ( + systemAttributePrefix = "__SYSTEM__" + + // deprecated: use systemAttributePrefix + systemAttributePrefixNeoFS = "__NEOFS__" +) + +type systemTransformer struct { + prefix string + backwardPrefix string + xAttrPrefixes [][]byte +} + +var transformers = []systemTransformer{ + { + prefix: systemAttributePrefix, + backwardPrefix: "System-", + xAttrPrefixes: [][]byte{[]byte("System-"), []byte("SYSTEM-"), []byte("system-")}, + }, + { + prefix: systemAttributePrefixNeoFS, + backwardPrefix: "Neofs-", + xAttrPrefixes: [][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")}, + }, +} + +func (t systemTransformer) existsExpirationAttributes(headers map[string]string) bool { + _, ok0 := headers[t.expirationEpochAttr()] + _, ok1 := headers[t.expirationDurationAttr()] + _, ok2 := headers[t.expirationTimestampAttr()] + _, ok3 := headers[t.expirationRFC3339Attr()] + return ok0 || ok1 || ok2 || ok3 +} + +func (t systemTransformer) expirationEpochAttr() string { + return t.prefix + "EXPIRATION_EPOCH" +} + +func (t systemTransformer) expirationDurationAttr() string { + return t.prefix + "EXPIRATION_DURATION" +} + +func (t systemTransformer) expirationTimestampAttr() string { + return t.prefix + "EXPIRATION_TIMESTAMP" +} + +func (t systemTransformer) expirationRFC3339Attr() string { + return t.prefix + "EXPIRATION_RFC3339" +} + +func (t systemTransformer) systemTranslator(key, prefix []byte) []byte { + // replace the specified prefix with system prefix + key = bytes.Replace(key, prefix, []byte(t.prefix), 1) + + // replace `-` with `_` + key = bytes.ReplaceAll(key, []byte("-"), []byte("_")) + + // replace with uppercase + return bytes.ToUpper(key) +} + +func (t systemTransformer) transformIfSystem(key []byte) ([]byte, bool) { + // checks that it's a system FrostFS header + for _, system := range t.xAttrPrefixes { + if bytes.HasPrefix(key, system) { + return t.systemTranslator(key, system), true + } + } + + return key, false +} + +// systemBackwardTranslator is used to convert headers looking like '__PREFIX__ATTR_NAME' to 'Prefix-Attr-Name'. +func (t systemTransformer) systemBackwardTranslator(key string) string { + // trim specified prefix '__PREFIX__' + key = strings.TrimPrefix(key, t.prefix) + + var res strings.Builder + res.WriteString(t.backwardPrefix) + + strs := strings.Split(key, "_") + for i, s := range strs { + s = title(strings.ToLower(s)) + res.WriteString(s) + if i != len(strs)-1 { + res.WriteString("-") + } + } + + return res.String() +} + +func (t systemTransformer) backwardTransformIfSystem(key string) (string, bool) { + if strings.HasPrefix(key, t.prefix) { + return t.systemBackwardTranslator(key), true + } + + return key, false +} + +func TransformIfSystem(key []byte) []byte { + for _, transformer := range transformers { + key, transformed := transformer.transformIfSystem(key) + if transformed { + return key + } + } + + return key +} + +func BackwardTransformIfSystem(key string) string { + for _, transformer := range transformers { + key, transformed := transformer.backwardTransformIfSystem(key) + if transformed { + return key + } + } + + return key +} + +func title(str string) string { + if str == "" { + return "" + } + + r, size := utf8.DecodeRuneInString(str) + r0 := unicode.ToTitle(r) + return string(r0) + str[size:] +} + +func PrepareExpirationHeader(ctx context.Context, p *pool.Pool, headers map[string]string, now time.Time) error { + formatsNum := 0 + index := -1 + for i, transformer := range transformers { + if transformer.existsExpirationAttributes(headers) { + formatsNum++ + index = i + } + } + + switch formatsNum { + case 0: + return nil + case 1: + epochDuration, err := GetEpochDurations(ctx, p) + if err != nil { + return fmt.Errorf("couldn't get epoch durations from network info: %w", err) + } + return transformers[index].prepareExpirationHeader(headers, epochDuration, now) + default: + return errors.New("both deprecated and new system attributes formats are used, please use only one") + } +} + +func (t systemTransformer) prepareExpirationHeader(headers map[string]string, epochDurations *EpochDurations, now time.Time) error { + expirationInEpoch := headers[t.expirationEpochAttr()] + + if timeRFC3339, ok := headers[t.expirationRFC3339Attr()]; ok { + expTime, err := time.Parse(time.RFC3339, timeRFC3339) + if err != nil { + return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, t.expirationRFC3339Attr()) + } + + if expTime.Before(now) { + return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, t.expirationRFC3339Attr()) + } + t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) + delete(headers, t.expirationRFC3339Attr()) + } + + if timestamp, ok := headers[t.expirationTimestampAttr()]; ok { + value, err := strconv.ParseInt(timestamp, 10, 64) + if err != nil { + return fmt.Errorf("couldn't parse value %s of header %s", timestamp, t.expirationTimestampAttr()) + } + expTime := time.Unix(value, 0) + + if expTime.Before(now) { + return fmt.Errorf("value %s of header %s must be in the future", timestamp, t.expirationTimestampAttr()) + } + t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) + delete(headers, t.expirationTimestampAttr()) + } + + if duration, ok := headers[t.expirationDurationAttr()]; ok { + expDuration, err := time.ParseDuration(duration) + if err != nil { + return fmt.Errorf("couldn't parse value %s of header %s", duration, t.expirationDurationAttr()) + } + if expDuration <= 0 { + return fmt.Errorf("value %s of header %s must be positive", expDuration, t.expirationDurationAttr()) + } + t.updateExpirationHeader(headers, epochDurations, expDuration) + delete(headers, t.expirationDurationAttr()) + } + + if expirationInEpoch != "" { + expEpoch, err := strconv.ParseUint(expirationInEpoch, 10, 64) + if err != nil { + return fmt.Errorf("parse expiration epoch '%s': %w", expirationInEpoch, err) + } + if expEpoch < epochDurations.CurrentEpoch { + return fmt.Errorf("expiration epoch '%d' must be greater than current epoch '%d'", expEpoch, epochDurations.CurrentEpoch) + } + + headers[t.expirationEpochAttr()] = expirationInEpoch + } + + return nil +} + +func (t systemTransformer) updateExpirationHeader(headers map[string]string, durations *EpochDurations, expDuration time.Duration) { + epochDuration := uint64(durations.MsPerBlock) * durations.BlockPerEpoch + currentEpoch := durations.CurrentEpoch + numEpoch := uint64(expDuration.Milliseconds()) / epochDuration + + if uint64(expDuration.Milliseconds())%epochDuration != 0 { + numEpoch++ + } + + expirationEpoch := uint64(math.MaxUint64) + if numEpoch < math.MaxUint64-currentEpoch { + expirationEpoch = currentEpoch + numEpoch + } + + headers[t.expirationEpochAttr()] = strconv.FormatUint(expirationEpoch, 10) +} diff --git a/utils/attributes_test.go b/utils/attributes_test.go new file mode 100644 index 0000000..a903b60 --- /dev/null +++ b/utils/attributes_test.go @@ -0,0 +1,187 @@ +package utils + +import ( + "math" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestPrepareExpirationHeader(t *testing.T) { + tomorrow := time.Now().Add(24 * time.Hour) + tomorrowUnix := tomorrow.Unix() + tomorrowUnixNano := tomorrow.UnixNano() + tomorrowUnixMilli := tomorrowUnixNano / 1e6 + + epoch := "100" + duration := "24h" + timestampSec := strconv.FormatInt(tomorrowUnix, 10) + timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10) + timestampNano := strconv.FormatInt(tomorrowUnixNano, 10) + + defaultDurations := &EpochDurations{ + CurrentEpoch: 10, + MsPerBlock: 1000, + BlockPerEpoch: 101, + } + + msPerBlock := defaultDurations.BlockPerEpoch * uint64(defaultDurations.MsPerBlock) + epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock + if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 { + epochPerDay++ + } + + defaultExpEpoch := strconv.FormatUint(defaultDurations.CurrentEpoch+epochPerDay, 10) + + for _, transformer := range transformers { + for _, tc := range []struct { + name string + headers map[string]string + durations *EpochDurations + err bool + expected map[string]string + }{ + { + name: "valid epoch", + headers: map[string]string{transformer.expirationEpochAttr(): epoch}, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + durations: defaultDurations, + }, + { + name: "valid epoch, valid duration", + headers: map[string]string{ + transformer.expirationEpochAttr(): epoch, + transformer.expirationDurationAttr(): duration, + }, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + }, + { + name: "valid epoch, valid rfc3339", + headers: map[string]string{ + transformer.expirationEpochAttr(): epoch, + transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339), + }, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + }, + { + name: "valid epoch, valid timestamp sec", + headers: map[string]string{ + transformer.expirationEpochAttr(): epoch, + transformer.expirationTimestampAttr(): timestampSec, + }, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + }, + { + name: "valid epoch, valid timestamp milli", + headers: map[string]string{ + transformer.expirationEpochAttr(): epoch, + transformer.expirationTimestampAttr(): timestampMilli, + }, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + }, + { + name: "valid epoch, valid timestamp nano", + headers: map[string]string{ + transformer.expirationEpochAttr(): epoch, + transformer.expirationTimestampAttr(): timestampNano, + }, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + }, + { + name: "valid timestamp sec", + headers: map[string]string{transformer.expirationTimestampAttr(): timestampSec}, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch}, + }, + { + name: "valid duration", + headers: map[string]string{transformer.expirationDurationAttr(): duration}, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch}, + }, + { + name: "valid rfc3339", + headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)}, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch}, + }, + { + name: "valid max uint 64", + headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)}, + durations: &EpochDurations{ + CurrentEpoch: math.MaxUint64 - 1, + MsPerBlock: defaultDurations.MsPerBlock, + BlockPerEpoch: defaultDurations.BlockPerEpoch, + }, + expected: map[string]string{transformer.expirationEpochAttr(): strconv.FormatUint(uint64(math.MaxUint64), 10)}, + }, + { + name: "invalid timestamp sec", + headers: map[string]string{transformer.expirationTimestampAttr(): "abc"}, + err: true, + }, + { + name: "invalid timestamp sec zero", + headers: map[string]string{transformer.expirationTimestampAttr(): "0"}, + err: true, + }, + { + name: "invalid duration", + headers: map[string]string{transformer.expirationDurationAttr(): "1d"}, + err: true, + }, + { + name: "invalid duration negative", + headers: map[string]string{transformer.expirationDurationAttr(): "-5h"}, + err: true, + }, + { + name: "invalid rfc3339", + headers: map[string]string{transformer.expirationRFC3339Attr(): "abc"}, + err: true, + }, + { + name: "invalid rfc3339 zero", + headers: map[string]string{transformer.expirationRFC3339Attr(): time.RFC3339}, + err: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := transformer.prepareExpirationHeader(tc.headers, tc.durations, time.Now()) + if tc.err { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.expected, tc.headers) + } + }) + } + } +} + +func TestSystemBackwardTranslator(t *testing.T) { + input := []string{ + "__SYSTEM__EXPIRATION_EPOCH", + "__SYSTEM__RANDOM_ATTR", + "__NEOFS__EXPIRATION_EPOCH", + "__NEOFS__RANDOM_ATTR", + } + expected := []string{ + "System-Expiration-Epoch", + "System-Random-Attr", + "Neofs-Expiration-Epoch", + "Neofs-Random-Attr", + } + + for i, str := range input { + res := BackwardTransformIfSystem(str) + require.Equal(t, expected[i], res) + } +} diff --git a/utils/util.go b/utils/util.go index a83064c..d5a476a 100644 --- a/utils/util.go +++ b/utils/util.go @@ -2,9 +2,11 @@ package utils import ( "context" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" ) // GetContainerID decode container id, if it's not a valid container id @@ -17,3 +19,27 @@ func GetContainerID(ctx context.Context, containerID string, resolver *resolver. } return cnrID, err } + +type EpochDurations struct { + CurrentEpoch uint64 + MsPerBlock int64 + BlockPerEpoch uint64 +} + +func GetEpochDurations(ctx context.Context, p *pool.Pool) (*EpochDurations, error) { + networkInfo, err := p.NetworkInfo(ctx) + if err != nil { + return nil, err + } + + res := &EpochDurations{ + CurrentEpoch: networkInfo.CurrentEpoch(), + MsPerBlock: networkInfo.MsPerBlock(), + BlockPerEpoch: networkInfo.EpochDuration(), + } + + if res.BlockPerEpoch == 0 { + return nil, fmt.Errorf("EpochDuration is empty") + } + return res, nil +}