TrueCloudLab/frostfs-http-gw
16
1
Fork 17
Code Issues 9 Pull requests 1 Projects Releases 9 Activity Actions

[#22] Update system attributes prefix

Browse source 
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
Denis Kirillov 2023-03-16 11:40:08 +03:00
parent 1f66149316
commit a8ec09e76a
14 changed files with 562 additions and 448 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
README.md
View file

@ -424,12 +424,12 @@ You can also add some attributes to your file using the following rules:
"X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo: "X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo:
100500" header to your request the resulting object will get "Ololo: 100500" header to your request the resulting object will get "Ololo:
100500" attribute 100500" attribute
* "X-Attribute-NEOFS-*" headers are special * "X-Attribute-SYSTEM-*" headers are special
(`-NEOFS-` part can also be `-neofs-` or`-Neofs-`), they're used to set internal (`-SYSTEM-` part can also be `-system-` or`-System-` (and even legacy `-Neofs-` for some next releases)), they're used to set internal
NeoFS attributes starting with `__NEOFS__` prefix, for these attributes all FrostFS attributes starting with `__SYSTEM__` prefix, for these attributes all
dashes get converted to underscores and all letters are capitalized. For dashes get converted to underscores and all letters are capitalized. For
example, you can use "X-Attribute-NEOFS-Expiration-Epoch" header to set example, you can use "X-Attribute-SYSTEM-Expiration-Epoch" header to set
`__NEOFS__EXPIRATION_EPOCH` attribute `__SYSTEM__EXPIRATION_EPOCH` attribute
* `FileName` attribute is set from multipart's `filename` if not set * `FileName` attribute is set from multipart's `filename` if not set
explicitly via `X-Attribute-FileName` header explicitly via `X-Attribute-FileName` header
* `Timestamp` attribute can be set using gateway local time if using * `Timestamp` attribute can be set using gateway local time if using
@ -439,13 +439,13 @@ You can also add some attributes to your file using the following rules:
--- ---
**NOTE** **NOTE**
There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority): There are some reserved headers type of `X-Attribute-SYSTEM-*` (headers are arranged in descending order of priority):
1. `X-Attribute-Neofs-Expiration-Epoch: 100` 1. `X-Attribute-System-Expiration-Epoch: 100`
2. `X-Attribute-Neofs-Expiration-Duration: 24h30m` 2. `X-Attribute-System-Expiration-Duration: 24h30m`
3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797` 3. `X-Attribute-System-Expiration-Timestamp: 1637574797`
4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z` 4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z`
which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way. which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way.
--- ---

108
docs/api.md
View file

@ -56,21 +56,21 @@ Upload file as object with attributes to FrostFS.
###### Headers ###### Headers
| Header | Description | | Header | Description |
|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------| |------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|
| Common headers | See [bearer token](#bearer-token). | | Common headers | See [bearer token](#bearer-token). |
| `X-Attribute-Neofs-*` | Used to set system NeoFS object attributes <br/> (e.g. use "X-Attribute-Neofs-Expiration-Epoch" to set `__NEOFS__EXPIRATION_EPOCH` attribute). | | `X-Attribute-System-*` | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). |
| `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | | `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). |
| `Date` | This header is used to calculate the right `__NEOFS__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | | `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. |
There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority): There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority):
1. `X-Attribute-Neofs-Expiration-Epoch: 100` 1. `X-Attribute-System-Expiration-Epoch: 100`
2. `X-Attribute-Neofs-Expiration-Duration: 24h30m` 2. `X-Attribute-System-Expiration-Duration: 24h30m`
3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797` 3. `X-Attribute-System-Expiration-Timestamp: 1637574797`
4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z` 4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z`
which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way. which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way.
If you don't specify the `X-Attribute-Timestamp` header the `Timestamp` attribute can be set anyway If you don't specify the `X-Attribute-Timestamp` header the `Timestamp` attribute can be set anyway
(see http-gw [configuration](gate-configuration.md#upload-header-section)). (see http-gw [configuration](gate-configuration.md#upload-header-section)).
@ -121,17 +121,17 @@ Get an object (payload and attributes) by an address.
###### Headers ###### Headers
| Header | Description | | Header | Description |
|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------| |------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | | `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). | | `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
| `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | | `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). |
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | | `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
| `Content-Length` | Size of object payload. | | `Content-Length` | Size of object payload. |
| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | | `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). |
| `X-Owner-Id` | Base58 encoded owner ID. | | `X-Owner-Id` | Base58 encoded owner ID. |
| `X-Container-Id` | Base58 encoded container ID. | | `X-Container-Id` | Base58 encoded container ID. |
| `X-Object-Id` | Base58 encoded object ID. | | `X-Object-Id` | Base58 encoded object ID. |
###### Status codes ###### Status codes
@ -157,16 +157,16 @@ Get an object attributes by an address.
###### Headers ###### Headers
| Header | Description | | Header | Description |
|-----------------------|--------------------------------------------------------------------------------------------------------------------------| |------------------------|------------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | | `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). | | `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | | `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
| `Content-Length` | Size of object payload. | | `Content-Length` | Size of object payload. |
| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | | `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). |
| `X-Owner-Id` | Base58 encoded owner ID. | | `X-Owner-Id` | Base58 encoded owner ID. |
| `X-Container-Id` | Base58 encoded container ID. | | `X-Container-Id` | Base58 encoded container ID. |
| `X-Object-Id` | Base58 encoded object ID. | | `X-Object-Id` | Base58 encoded object ID. |
###### Status codes ###### Status codes
@ -206,17 +206,17 @@ If more than one object is found, an arbitrary one will be returned.
###### Headers ###### Headers
| Header | Description | | Header | Description |
|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------| |------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | | `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). | | `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
| `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | | `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). |
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | | `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
| `Content-Length` | Size of object payload. | | `Content-Length` | Size of object payload. |
| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | | `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). |
| `X-Owner-Id` | Base58 encoded owner ID. | | `X-Owner-Id` | Base58 encoded owner ID. |
| `X-Container-Id` | Base58 encoded container ID. | | `X-Container-Id` | Base58 encoded container ID. |
| `X-Object-Id` | Base58 encoded object ID. | | `X-Object-Id` | Base58 encoded object ID. |
###### Status codes ###### Status codes
@ -243,16 +243,16 @@ If more than one object is found, an arbitrary one will be used to get attribute
###### Headers ###### Headers
| Header | Description | | Header | Description |
|-----------------------|--------------------------------------------------------------------------------------------------------------------------| |------------------------|------------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | | `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). | | `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | | `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
| `Content-Length` | Size of object payload. | | `Content-Length` | Size of object payload. |
| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | | `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). |
| `X-Owner-Id` | Base58 encoded owner ID. | | `X-Owner-Id` | Base58 encoded owner ID. |
| `X-Container-Id` | Base58 encoded container ID. | | `X-Container-Id` | Base58 encoded container ID. |
| `X-Object-Id` | Base58 encoded object ID. | | `X-Object-Id` | Base58 encoded object ID. |
###### Status codes ###### Status codes

38
downloader/download.go
View file

@ -14,8 +14,6 @@ import (
"strconv" "strconv"
"strings" "strings"
"time" "time"
"unicode"
"unicode/utf8"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
@ -131,9 +129,9 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) {
if !isValidToken(key) || !isValidValue(val) { if !isValidToken(key) || !isValidValue(val) {
continue continue
} }
if strings.HasPrefix(key, utils.SystemAttributePrefix) {
key = systemBackwardTranslator(key) key = utils.BackwardTransformIfSystem(key)
}
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
switch key { switch key {
case object.AttributeFileName: case object.AttributeFileName:
@ -187,36 +185,6 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) {
r.Response.SetBodyStream(rObj.Payload, int(payloadSize)) r.Response.SetBodyStream(rObj.Payload, int(payloadSize))
} }
// systemBackwardTranslator is used to convert headers looking like '__NEOFS__ATTR_NAME' to 'Neofs-Attr-Name'.
func systemBackwardTranslator(key string) string {
// trim specified prefix '__NEOFS__'
key = strings.TrimPrefix(key, utils.SystemAttributePrefix)
var res strings.Builder
res.WriteString("Neofs-")
strs := strings.Split(key, "_")
for i, s := range strs {
s = title(strings.ToLower(s))
res.WriteString(s)
if i != len(strs)-1 {
res.WriteString("-")
}
}
return res.String()
}
func title(str string) string {
if str == "" {
return ""
}
r, size := utf8.DecodeRuneInString(str)
r0 := unicode.ToTitle(r)
return string(r0) + str[size:]
}
func bearerToken(ctx context.Context) *bearer.Token { func bearerToken(ctx context.Context) *bearer.Token {
if tkn, err := tokens.LoadBearerToken(ctx); err == nil { if tkn, err := tokens.LoadBearerToken(ctx); err == nil {
return tkn return tkn

23
downloader/download_test.go
View file

@ -1,23 +0,0 @@
package downloader
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestSystemBackwardTranslator(t *testing.T) {
input := []string{
"__NEOFS__EXPIRATION_EPOCH",
"__NEOFS__RANDOM_ATTR",
}
expected := []string{
"Neofs-Expiration-Epoch",
"Neofs-Random-Attr",
}
for i, str := range input {
res := systemBackwardTranslator(str)
require.Equal(t, expected[i], res)
}
}

7
downloader/head.go
View file

@ -4,7 +4,6 @@ import (
"io" "io"
"net/http" "net/http"
"strconv" "strconv"
"strings"
"time" "time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
@ -56,9 +55,9 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) {
if !isValidToken(key) || !isValidValue(val) { if !isValidToken(key) || !isValidValue(val) {
continue continue
} }
if strings.HasPrefix(key, utils.SystemAttributePrefix) {
key = systemBackwardTranslator(key) key = utils.BackwardTransformIfSystem(key)
}
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
switch key { switch key {
case object.AttributeTimestamp: case object.AttributeTimestamp:

4
go.mod
View file

@ -3,8 +3,8 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw
go 1.18 go 1.18
require ( require (
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85
github.com/fasthttp/router v1.4.1 github.com/fasthttp/router v1.4.1
github.com/nspcc-dev/neo-go v0.101.0 github.com/nspcc-dev/neo-go v0.101.0
github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_golang v1.13.0

8
go.sum
View file

@ -37,14 +37,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51 h1:l4+K1hN+NuWNtlZZoV8yRRP3Uu7PifL05ukEqKcb0Ks= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 h1:lxe0DtZq/uFZVZu9apx6OcIXCJskQBMd/GVeYGKA3wA=
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51/go.mod h1:n0DxKYulu2Ar73R6OcNF34LiL/Xa+iDR7GZuaOChbLE= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703/go.mod h1:gRd5iE5A84viily6AcNBsSlTx2XgoWrwRDz7z0MayDQ=
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M=
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o=
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599 h1:mzGX2RX8R8H/tUqrUu1TcYk4QRDBcBIWGYscPncfLOQ= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 h1:TUcJ5A0C1gWi3bAhw4b+V+iVM3E9mbBOdJIWWkAPNxo=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599/go.mod h1:z7zcpGY+puI5puyy5oyFbf20vWp84WtslCxcr6/kv5c= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85/go.mod h1:23fUGlEv/ImaOi3vck6vZj0v0b4hteOhLLPnVWHSQeA=
git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk=
git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y=
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA=

7
integration_test.go
View file

@ -13,6 +13,7 @@ import (
"testing" "testing"
"time" "time"
containerv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -407,7 +408,11 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o
if version >= versionWithNativeNames { if version >= versionWithNativeNames {
var domain container.Domain var domain container.Domain
domain.SetName(testContainerName) domain.SetName(testContainerName)
container.WriteDomain(&cnr, domain)
// currently node in aio image knows nothing about new sys attributes
// todo (@dkirillov): #2 use frostfs aio images that supports new attributes
cnr.SetAttribute(containerv2.SysAttributeNameNeoFS, domain.Name())
cnr.SetAttribute(containerv2.SysAttributeZoneNeoFS, domain.Zone())
} }
var waitPrm pool.WaitParams var waitPrm pool.WaitParams

91
uploader/filter.go
View file

@ -3,29 +3,12 @@ package uploader
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"math"
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"go.uber.org/zap" "go.uber.org/zap"
) )
var frostfsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")}
func systemTranslator(key, prefix []byte) []byte {
// replace the specified prefix with `__NEOFS__`
key = bytes.Replace(key, prefix, []byte(utils.SystemAttributePrefix), 1)
// replace `-` with `_`
key = bytes.ReplaceAll(key, []byte("-"), []byte("_"))
// replace with uppercase
return bytes.ToUpper(key)
}
func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]string, error) { func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]string, error) {
var err error var err error
result := make(map[string]string) result := make(map[string]string)
@ -45,13 +28,7 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st
// removing attribute prefix // removing attribute prefix
clearKey := bytes.TrimPrefix(key, prefix) clearKey := bytes.TrimPrefix(key, prefix)
// checks that it's a system NeoFS header clearKey = utils.TransformIfSystem(clearKey)
for _, system := range frostfsAttributeHeaderPrefixes {
if bytes.HasPrefix(clearKey, system) {
clearKey = systemTranslator(clearKey, system)
break
}
}
// checks that the attribute key is not empty // checks that the attribute key is not empty
if len(clearKey) == 0 { if len(clearKey) == 0 {
@ -77,69 +54,3 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st
return result, err return result, err
} }
func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations, now time.Time) error {
expirationInEpoch := headers[object.SysAttributeExpEpoch]
if timeRFC3339, ok := headers[utils.ExpirationRFC3339Attr]; ok {
expTime, err := time.Parse(time.RFC3339, timeRFC3339)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, utils.ExpirationRFC3339Attr)
}
if expTime.Before(now) {
return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, utils.ExpirationRFC3339Attr)
}
updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
delete(headers, utils.ExpirationRFC3339Attr)
}
if timestamp, ok := headers[utils.ExpirationTimestampAttr]; ok {
value, err := strconv.ParseInt(timestamp, 10, 64)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", timestamp, utils.ExpirationTimestampAttr)
}
expTime := time.Unix(value, 0)
if expTime.Before(now) {
return fmt.Errorf("value %s of header %s must be in the future", timestamp, utils.ExpirationTimestampAttr)
}
updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
delete(headers, utils.ExpirationTimestampAttr)
}
if duration, ok := headers[utils.ExpirationDurationAttr]; ok {
expDuration, err := time.ParseDuration(duration)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", duration, utils.ExpirationDurationAttr)
}
if expDuration <= 0 {
return fmt.Errorf("value %s of header %s must be positive", expDuration, utils.ExpirationDurationAttr)
}
updateExpirationHeader(headers, epochDurations, expDuration)
delete(headers, utils.ExpirationDurationAttr)
}
if expirationInEpoch != "" {
headers[object.SysAttributeExpEpoch] = expirationInEpoch
}
return nil
}
func updateExpirationHeader(headers map[string]string, durations *epochDurations, expDuration time.Duration) {
epochDuration := uint64(durations.msPerBlock) * durations.blockPerEpoch
currentEpoch := durations.currentEpoch
numEpoch := uint64(expDuration.Milliseconds()) / epochDuration
if uint64(expDuration.Milliseconds())%epochDuration != 0 {
numEpoch++
}
expirationEpoch := uint64(math.MaxUint64)
if numEpoch < math.MaxUint64-currentEpoch {
expirationEpoch = currentEpoch + numEpoch
}
headers[object.SysAttributeExpEpoch] = strconv.FormatUint(expirationEpoch, 10)
}

177
uploader/filter_test.go
View file

@ -1,13 +1,8 @@
package uploader package uploader
import ( import (
"math"
"strconv"
"testing" "testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"go.uber.org/zap" "go.uber.org/zap"
@ -28,8 +23,8 @@ func TestFilter(t *testing.T) {
t.Run("duplicate system keys error", func(t *testing.T) { t.Run("duplicate system keys error", func(t *testing.T) {
req := &fasthttp.RequestHeader{} req := &fasthttp.RequestHeader{}
req.DisableNormalizing() req.DisableNormalizing()
req.Add("X-Attribute-Neofs-DupKey", "first-value") req.Add("X-Attribute-System-DupKey", "first-value")
req.Add("X-Attribute-Neofs-DupKey", "second-value") req.Add("X-Attribute-System-DupKey", "second-value")
_, err := filterHeaders(log, req) _, err := filterHeaders(log, req)
require.Error(t, err) require.Error(t, err)
}) })
@ -37,16 +32,16 @@ func TestFilter(t *testing.T) {
req := &fasthttp.RequestHeader{} req := &fasthttp.RequestHeader{}
req.DisableNormalizing() req.DisableNormalizing()
req.Set("X-Attribute-Neofs-Expiration-Epoch1", "101") req.Set("X-Attribute-System-Expiration-Epoch1", "101")
req.Set("X-Attribute-NEOFS-Expiration-Epoch2", "102") req.Set("X-Attribute-SYSTEM-Expiration-Epoch2", "102")
req.Set("X-Attribute-neofs-Expiration-Epoch3", "103") req.Set("X-Attribute-system-Expiration-Epoch3", "103")
req.Set("X-Attribute-MyAttribute", "value") req.Set("X-Attribute-MyAttribute", "value")
expected := map[string]string{ expected := map[string]string{
"__NEOFS__EXPIRATION_EPOCH1": "101", "__SYSTEM__EXPIRATION_EPOCH1": "101",
"MyAttribute": "value", "MyAttribute": "value",
"__NEOFS__EXPIRATION_EPOCH3": "103", "__SYSTEM__EXPIRATION_EPOCH3": "103",
"__NEOFS__EXPIRATION_EPOCH2": "102", "__SYSTEM__EXPIRATION_EPOCH2": "102",
} }
result, err := filterHeaders(log, req) result, err := filterHeaders(log, req)
@ -54,157 +49,3 @@ func TestFilter(t *testing.T) {
require.Equal(t, expected, result) require.Equal(t, expected, result)
} }
func TestPrepareExpirationHeader(t *testing.T) {
tomorrow := time.Now().Add(24 * time.Hour)
tomorrowUnix := tomorrow.Unix()
tomorrowUnixNano := tomorrow.UnixNano()
tomorrowUnixMilli := tomorrowUnixNano / 1e6
epoch := "100"
duration := "24h"
timestampSec := strconv.FormatInt(tomorrowUnix, 10)
timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10)
timestampNano := strconv.FormatInt(tomorrowUnixNano, 10)
defaultDurations := &epochDurations{
currentEpoch: 10,
msPerBlock: 1000,
blockPerEpoch: 101,
}
msPerBlock := defaultDurations.blockPerEpoch * uint64(defaultDurations.msPerBlock)
epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock
if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 {
epochPerDay++
}
defaultExpEpoch := strconv.FormatUint(defaultDurations.currentEpoch+epochPerDay, 10)
for _, tc := range []struct {
name string
headers map[string]string
durations *epochDurations
err bool
expected map[string]string
}{
{
name: "valid epoch",
headers: map[string]string{object.SysAttributeExpEpoch: epoch},
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid epoch, valid duration",
headers: map[string]string{
object.SysAttributeExpEpoch: epoch,
utils.ExpirationDurationAttr: duration,
},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid epoch, valid rfc3339",
headers: map[string]string{
object.SysAttributeExpEpoch: epoch,
utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339),
},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid epoch, valid timestamp sec",
headers: map[string]string{
object.SysAttributeExpEpoch: epoch,
utils.ExpirationTimestampAttr: timestampSec,
},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid epoch, valid timestamp milli",
headers: map[string]string{
object.SysAttributeExpEpoch: epoch,
utils.ExpirationTimestampAttr: timestampMilli,
},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid epoch, valid timestamp nano",
headers: map[string]string{
object.SysAttributeExpEpoch: epoch,
utils.ExpirationTimestampAttr: timestampNano,
},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid timestamp sec",
headers: map[string]string{utils.ExpirationTimestampAttr: timestampSec},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
},
{
name: "valid duration",
headers: map[string]string{utils.ExpirationDurationAttr: duration},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
},
{
name: "valid rfc3339",
headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
},
{
name: "valid max uint 64",
headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)},
durations: &epochDurations{
currentEpoch: math.MaxUint64 - 1,
msPerBlock: defaultDurations.msPerBlock,
blockPerEpoch: defaultDurations.blockPerEpoch,
},
expected: map[string]string{object.SysAttributeExpEpoch: strconv.FormatUint(uint64(math.MaxUint64), 10)},
},
{
name: "invalid timestamp sec",
headers: map[string]string{utils.ExpirationTimestampAttr: "abc"},
err: true,
},
{
name: "invalid timestamp sec zero",
headers: map[string]string{utils.ExpirationTimestampAttr: "0"},
err: true,
},
{
name: "invalid duration",
headers: map[string]string{utils.ExpirationDurationAttr: "1d"},
err: true,
},
{
name: "invalid duration negative",
headers: map[string]string{utils.ExpirationDurationAttr: "-5h"},
err: true,
},
{
name: "invalid rfc3339",
headers: map[string]string{utils.ExpirationRFC3339Attr: "abc"},
err: true,
},
{
name: "invalid rfc3339 zero",
headers: map[string]string{utils.ExpirationRFC3339Attr: time.RFC3339},
err: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
err := prepareExpirationHeader(tc.headers, tc.durations, time.Now())
if tc.err {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tc.expected, tc.headers)
}
})
}
}

62
uploader/upload.go
View file

@ -3,7 +3,6 @@ package uploader
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"io" "io"
"net/http" "net/http"
"strconv" "strconv"
@ -38,12 +37,6 @@ type Uploader struct {
containerResolver *resolver.ContainerResolver containerResolver *resolver.ContainerResolver
} }
type epochDurations struct {
currentEpoch uint64
msPerBlock int64
blockPerEpoch uint64
}
// Settings stores reloading parameters, so it has to provide atomic getters and setters. // Settings stores reloading parameters, so it has to provide atomic getters and setters.
type Settings struct { type Settings struct {
defaultTimestamp atomic.Bool defaultTimestamp atomic.Bool
@ -120,28 +113,20 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
response.Error(c, err.Error(), fasthttp.StatusBadRequest) response.Error(c, err.Error(), fasthttp.StatusBadRequest)
return return
} }
if needParseExpiration(filtered) {
epochDuration, err := getEpochDurations(c, u.pool)
if err != nil {
log.Error("could not get epoch durations from network info", zap.Error(err))
response.Error(c, "could not get epoch durations from network info: "+err.Error(), fasthttp.StatusBadRequest)
return
}
now := time.Now() now := time.Now()
if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil {
if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil {
log.Warn("could not parse client time", zap.String("Date header", string(rawHeader)), zap.Error(err)) log.Warn("could not parse client time", zap.String("Date header", string(rawHeader)), zap.Error(err))
} else { } else {
now = parsed now = parsed
}
} }
}
if err = prepareExpirationHeader(filtered, epochDuration, now); err != nil { if err = utils.PrepareExpirationHeader(c, u.pool, filtered, now); err != nil {
log.Error("could not parse expiration header", zap.Error(err)) log.Error("could not prepare expiration header", zap.Error(err))
response.Error(c, "could not parse expiration header: "+err.Error(), fasthttp.StatusBadRequest) response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest)
return return
}
} }
attributes := make([]object.Attribute, 0, len(filtered)) attributes := make([]object.Attribute, 0, len(filtered))
@ -246,28 +231,3 @@ func (pr *putResponse) encode(w io.Writer) error {
enc.SetIndent("", "\t") enc.SetIndent("", "\t")
return enc.Encode(pr) return enc.Encode(pr)
} }
func getEpochDurations(ctx context.Context, p *pool.Pool) (*epochDurations, error) {
networkInfo, err := p.NetworkInfo(ctx)
if err != nil {
return nil, err
}
res := &epochDurations{
currentEpoch: networkInfo.CurrentEpoch(),
msPerBlock: networkInfo.MsPerBlock(),
blockPerEpoch: networkInfo.EpochDuration(),
}
if res.blockPerEpoch == 0 {
return nil, fmt.Errorf("EpochDuration is empty")
}
return res, nil
}
func needParseExpiration(headers map[string]string) bool {
_, ok1 := headers[utils.ExpirationDurationAttr]
_, ok2 := headers[utils.ExpirationRFC3339Attr]
_, ok3 := headers[utils.ExpirationTimestampAttr]
return ok1 || ok2 || ok3
}

250
utils/attributes.go
View file

@ -1,10 +1,250 @@
package utils package utils
import (
"bytes"
"context"
"errors"
"fmt"
"math"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
)
const ( const (
UserAttributeHeaderPrefix = "X-Attribute-" UserAttributeHeaderPrefix = "X-Attribute-"
SystemAttributePrefix = "__NEOFS__"
ExpirationDurationAttr = SystemAttributePrefix + "EXPIRATION_DURATION"
ExpirationTimestampAttr = SystemAttributePrefix + "EXPIRATION_TIMESTAMP"
ExpirationRFC3339Attr = SystemAttributePrefix + "EXPIRATION_RFC3339"
) )
const (
systemAttributePrefix = "__SYSTEM__"
// deprecated: use systemAttributePrefix
systemAttributePrefixNeoFS = "__NEOFS__"
)
type systemTransformer struct {
prefix string
backwardPrefix string
xAttrPrefixes [][]byte
}
var transformers = []systemTransformer{
{
prefix: systemAttributePrefix,
backwardPrefix: "System-",
xAttrPrefixes: [][]byte{[]byte("System-"), []byte("SYSTEM-"), []byte("system-")},
},
{
prefix: systemAttributePrefixNeoFS,
backwardPrefix: "Neofs-",
xAttrPrefixes: [][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")},
},
}
func (t systemTransformer) existsExpirationAttributes(headers map[string]string) bool {
_, ok0 := headers[t.expirationEpochAttr()]
_, ok1 := headers[t.expirationDurationAttr()]
_, ok2 := headers[t.expirationTimestampAttr()]
_, ok3 := headers[t.expirationRFC3339Attr()]
return ok0 || ok1 || ok2 || ok3
}
func (t systemTransformer) expirationEpochAttr() string {
return t.prefix + "EXPIRATION_EPOCH"
}
func (t systemTransformer) expirationDurationAttr() string {
return t.prefix + "EXPIRATION_DURATION"
}
func (t systemTransformer) expirationTimestampAttr() string {
return t.prefix + "EXPIRATION_TIMESTAMP"
}
func (t systemTransformer) expirationRFC3339Attr() string {
return t.prefix + "EXPIRATION_RFC3339"
}
func (t systemTransformer) systemTranslator(key, prefix []byte) []byte {
// replace the specified prefix with system prefix
key = bytes.Replace(key, prefix, []byte(t.prefix), 1)
// replace `-` with `_`
key = bytes.ReplaceAll(key, []byte("-"), []byte("_"))
// replace with uppercase
return bytes.ToUpper(key)
}
func (t systemTransformer) transformIfSystem(key []byte) ([]byte, bool) {
// checks that it's a system FrostFS header
for _, system := range t.xAttrPrefixes {
if bytes.HasPrefix(key, system) {
return t.systemTranslator(key, system), true
}
}
return key, false
}
// systemBackwardTranslator is used to convert headers looking like '__PREFIX__ATTR_NAME' to 'Prefix-Attr-Name'.
func (t systemTransformer) systemBackwardTranslator(key string) string {
// trim specified prefix '__PREFIX__'
key = strings.TrimPrefix(key, t.prefix)
var res strings.Builder
res.WriteString(t.backwardPrefix)
strs := strings.Split(key, "_")
for i, s := range strs {
s = title(strings.ToLower(s))
res.WriteString(s)
if i != len(strs)-1 {
res.WriteString("-")
}
}
return res.String()
}
func (t systemTransformer) backwardTransformIfSystem(key string) (string, bool) {
if strings.HasPrefix(key, t.prefix) {
return t.systemBackwardTranslator(key), true
}
return key, false
}
func TransformIfSystem(key []byte) []byte {
for _, transformer := range transformers {
key, transformed := transformer.transformIfSystem(key)
if transformed {
return key
}
}
return key
}
func BackwardTransformIfSystem(key string) string {
for _, transformer := range transformers {
key, transformed := transformer.backwardTransformIfSystem(key)
if transformed {
return key
}
}
return key
}
func title(str string) string {
if str == "" {
return ""
}
r, size := utf8.DecodeRuneInString(str)
r0 := unicode.ToTitle(r)
return string(r0) + str[size:]
}
func PrepareExpirationHeader(ctx context.Context, p *pool.Pool, headers map[string]string, now time.Time) error {
formatsNum := 0
index := -1
for i, transformer := range transformers {
if transformer.existsExpirationAttributes(headers) {
formatsNum++
index = i
}
}
switch formatsNum {
case 0:
return nil
case 1:
epochDuration, err := GetEpochDurations(ctx, p)
if err != nil {
return fmt.Errorf("couldn't get epoch durations from network info: %w", err)
}
return transformers[index].prepareExpirationHeader(headers, epochDuration, now)
default:
return errors.New("both deprecated and new system attributes formats are used, please use only one")
}
}
func (t systemTransformer) prepareExpirationHeader(headers map[string]string, epochDurations *EpochDurations, now time.Time) error {
expirationInEpoch := headers[t.expirationEpochAttr()]
if timeRFC3339, ok := headers[t.expirationRFC3339Attr()]; ok {
expTime, err := time.Parse(time.RFC3339, timeRFC3339)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, t.expirationRFC3339Attr())
}
if expTime.Before(now) {
return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, t.expirationRFC3339Attr())
}
t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
delete(headers, t.expirationRFC3339Attr())
}
if timestamp, ok := headers[t.expirationTimestampAttr()]; ok {
value, err := strconv.ParseInt(timestamp, 10, 64)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", timestamp, t.expirationTimestampAttr())
}
expTime := time.Unix(value, 0)
if expTime.Before(now) {
return fmt.Errorf("value %s of header %s must be in the future", timestamp, t.expirationTimestampAttr())
}
t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
delete(headers, t.expirationTimestampAttr())
}
if duration, ok := headers[t.expirationDurationAttr()]; ok {
expDuration, err := time.ParseDuration(duration)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", duration, t.expirationDurationAttr())
}
if expDuration <= 0 {
return fmt.Errorf("value %s of header %s must be positive", expDuration, t.expirationDurationAttr())
}
t.updateExpirationHeader(headers, epochDurations, expDuration)
delete(headers, t.expirationDurationAttr())
}
if expirationInEpoch != "" {
expEpoch, err := strconv.ParseUint(expirationInEpoch, 10, 64)
if err != nil {
return fmt.Errorf("parse expiration epoch '%s': %w", expirationInEpoch, err)
}
if expEpoch < epochDurations.CurrentEpoch {
return fmt.Errorf("expiration epoch '%d' must be greater than current epoch '%d'", expEpoch, epochDurations.CurrentEpoch)
}
headers[t.expirationEpochAttr()] = expirationInEpoch
}
return nil
}
func (t systemTransformer) updateExpirationHeader(headers map[string]string, durations *EpochDurations, expDuration time.Duration) {
epochDuration := uint64(durations.MsPerBlock) * durations.BlockPerEpoch
currentEpoch := durations.CurrentEpoch
numEpoch := uint64(expDuration.Milliseconds()) / epochDuration
if uint64(expDuration.Milliseconds())%epochDuration != 0 {
numEpoch++
}
expirationEpoch := uint64(math.MaxUint64)
if numEpoch < math.MaxUint64-currentEpoch {
expirationEpoch = currentEpoch + numEpoch
}
headers[t.expirationEpochAttr()] = strconv.FormatUint(expirationEpoch, 10)
}

187
utils/attributes_test.go Normal file
View file

@ -0,0 +1,187 @@
package utils
import (
"math"
"strconv"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestPrepareExpirationHeader(t *testing.T) {
tomorrow := time.Now().Add(24 * time.Hour)
tomorrowUnix := tomorrow.Unix()
tomorrowUnixNano := tomorrow.UnixNano()
tomorrowUnixMilli := tomorrowUnixNano / 1e6
epoch := "100"
duration := "24h"
timestampSec := strconv.FormatInt(tomorrowUnix, 10)
timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10)
timestampNano := strconv.FormatInt(tomorrowUnixNano, 10)
defaultDurations := &EpochDurations{
CurrentEpoch: 10,
MsPerBlock: 1000,
BlockPerEpoch: 101,
}
msPerBlock := defaultDurations.BlockPerEpoch * uint64(defaultDurations.MsPerBlock)
epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock
if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 {
epochPerDay++
}
defaultExpEpoch := strconv.FormatUint(defaultDurations.CurrentEpoch+epochPerDay, 10)
for _, transformer := range transformers {
for _, tc := range []struct {
name string
headers map[string]string
durations *EpochDurations
err bool
expected map[string]string
}{
{
name: "valid epoch",
headers: map[string]string{transformer.expirationEpochAttr(): epoch},
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
durations: defaultDurations,
},
{
name: "valid epoch, valid duration",
headers: map[string]string{
transformer.expirationEpochAttr(): epoch,
transformer.expirationDurationAttr(): duration,
},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
},
{
name: "valid epoch, valid rfc3339",
headers: map[string]string{
transformer.expirationEpochAttr(): epoch,
transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339),
},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
},
{
name: "valid epoch, valid timestamp sec",
headers: map[string]string{
transformer.expirationEpochAttr(): epoch,
transformer.expirationTimestampAttr(): timestampSec,
},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
},
{
name: "valid epoch, valid timestamp milli",
headers: map[string]string{
transformer.expirationEpochAttr(): epoch,
transformer.expirationTimestampAttr(): timestampMilli,
},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
},
{
name: "valid epoch, valid timestamp nano",
headers: map[string]string{
transformer.expirationEpochAttr(): epoch,
transformer.expirationTimestampAttr(): timestampNano,
},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
},
{
name: "valid timestamp sec",
headers: map[string]string{transformer.expirationTimestampAttr(): timestampSec},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
},
{
name: "valid duration",
headers: map[string]string{transformer.expirationDurationAttr(): duration},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
},
{
name: "valid rfc3339",
headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
},
{
name: "valid max uint 64",
headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)},
durations: &EpochDurations{
CurrentEpoch: math.MaxUint64 - 1,
MsPerBlock: defaultDurations.MsPerBlock,
BlockPerEpoch: defaultDurations.BlockPerEpoch,
},
expected: map[string]string{transformer.expirationEpochAttr(): strconv.FormatUint(uint64(math.MaxUint64), 10)},
},
{
name: "invalid timestamp sec",
headers: map[string]string{transformer.expirationTimestampAttr(): "abc"},
err: true,
},
{
name: "invalid timestamp sec zero",
headers: map[string]string{transformer.expirationTimestampAttr(): "0"},
err: true,
},
{
name: "invalid duration",
headers: map[string]string{transformer.expirationDurationAttr(): "1d"},
err: true,
},
{
name: "invalid duration negative",
headers: map[string]string{transformer.expirationDurationAttr(): "-5h"},
err: true,
},
{
name: "invalid rfc3339",
headers: map[string]string{transformer.expirationRFC3339Attr(): "abc"},
err: true,
},
{
name: "invalid rfc3339 zero",
headers: map[string]string{transformer.expirationRFC3339Attr(): time.RFC3339},
err: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
err := transformer.prepareExpirationHeader(tc.headers, tc.durations, time.Now())
if tc.err {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tc.expected, tc.headers)
}
})
}
}
}
func TestSystemBackwardTranslator(t *testing.T) {
input := []string{
"__SYSTEM__EXPIRATION_EPOCH",
"__SYSTEM__RANDOM_ATTR",
"__NEOFS__EXPIRATION_EPOCH",
"__NEOFS__RANDOM_ATTR",
}
expected := []string{
"System-Expiration-Epoch",
"System-Random-Attr",
"Neofs-Expiration-Epoch",
"Neofs-Random-Attr",
}
for i, str := range input {
res := BackwardTransformIfSystem(str)
require.Equal(t, expected[i], res)
}
}

26
utils/util.go
View file

@ -2,9 +2,11 @@ package utils
import ( import (
"context" "context"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
) )
// GetContainerID decode container id, if it's not a valid container id // GetContainerID decode container id, if it's not a valid container id
@ -17,3 +19,27 @@ func GetContainerID(ctx context.Context, containerID string, resolver *resolver.
} }
return cnrID, err return cnrID, err
} }
type EpochDurations struct {
CurrentEpoch uint64
MsPerBlock int64
BlockPerEpoch uint64
}
func GetEpochDurations(ctx context.Context, p *pool.Pool) (*EpochDurations, error) {
networkInfo, err := p.NetworkInfo(ctx)
if err != nil {
return nil, err
}
res := &EpochDurations{
CurrentEpoch: networkInfo.CurrentEpoch(),
MsPerBlock: networkInfo.MsPerBlock(),
BlockPerEpoch: networkInfo.EpochDuration(),
}
if res.BlockPerEpoch == 0 {
return nil, fmt.Errorf("EpochDuration is empty")
}
return res, nil
}