forked from TrueCloudLab/frostfs-http-gw
[#22] Update system attributes prefix
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
parent
1f66149316
commit
a8ec09e76a
14 changed files with 562 additions and 448 deletions
|
|
@ -1,10 +1,250 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
)
|
||||
|
||||
const (
|
||||
UserAttributeHeaderPrefix = "X-Attribute-"
|
||||
SystemAttributePrefix = "__NEOFS__"
|
||||
|
||||
ExpirationDurationAttr = SystemAttributePrefix + "EXPIRATION_DURATION"
|
||||
ExpirationTimestampAttr = SystemAttributePrefix + "EXPIRATION_TIMESTAMP"
|
||||
ExpirationRFC3339Attr = SystemAttributePrefix + "EXPIRATION_RFC3339"
|
||||
)
|
||||
|
||||
const (
|
||||
systemAttributePrefix = "__SYSTEM__"
|
||||
|
||||
// deprecated: use systemAttributePrefix
|
||||
systemAttributePrefixNeoFS = "__NEOFS__"
|
||||
)
|
||||
|
||||
type systemTransformer struct {
|
||||
prefix string
|
||||
backwardPrefix string
|
||||
xAttrPrefixes [][]byte
|
||||
}
|
||||
|
||||
var transformers = []systemTransformer{
|
||||
{
|
||||
prefix: systemAttributePrefix,
|
||||
backwardPrefix: "System-",
|
||||
xAttrPrefixes: [][]byte{[]byte("System-"), []byte("SYSTEM-"), []byte("system-")},
|
||||
},
|
||||
{
|
||||
prefix: systemAttributePrefixNeoFS,
|
||||
backwardPrefix: "Neofs-",
|
||||
xAttrPrefixes: [][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")},
|
||||
},
|
||||
}
|
||||
|
||||
func (t systemTransformer) existsExpirationAttributes(headers map[string]string) bool {
|
||||
_, ok0 := headers[t.expirationEpochAttr()]
|
||||
_, ok1 := headers[t.expirationDurationAttr()]
|
||||
_, ok2 := headers[t.expirationTimestampAttr()]
|
||||
_, ok3 := headers[t.expirationRFC3339Attr()]
|
||||
return ok0 || ok1 || ok2 || ok3
|
||||
}
|
||||
|
||||
func (t systemTransformer) expirationEpochAttr() string {
|
||||
return t.prefix + "EXPIRATION_EPOCH"
|
||||
}
|
||||
|
||||
func (t systemTransformer) expirationDurationAttr() string {
|
||||
return t.prefix + "EXPIRATION_DURATION"
|
||||
}
|
||||
|
||||
func (t systemTransformer) expirationTimestampAttr() string {
|
||||
return t.prefix + "EXPIRATION_TIMESTAMP"
|
||||
}
|
||||
|
||||
func (t systemTransformer) expirationRFC3339Attr() string {
|
||||
return t.prefix + "EXPIRATION_RFC3339"
|
||||
}
|
||||
|
||||
func (t systemTransformer) systemTranslator(key, prefix []byte) []byte {
|
||||
// replace the specified prefix with system prefix
|
||||
key = bytes.Replace(key, prefix, []byte(t.prefix), 1)
|
||||
|
||||
// replace `-` with `_`
|
||||
key = bytes.ReplaceAll(key, []byte("-"), []byte("_"))
|
||||
|
||||
// replace with uppercase
|
||||
return bytes.ToUpper(key)
|
||||
}
|
||||
|
||||
func (t systemTransformer) transformIfSystem(key []byte) ([]byte, bool) {
|
||||
// checks that it's a system FrostFS header
|
||||
for _, system := range t.xAttrPrefixes {
|
||||
if bytes.HasPrefix(key, system) {
|
||||
return t.systemTranslator(key, system), true
|
||||
}
|
||||
}
|
||||
|
||||
return key, false
|
||||
}
|
||||
|
||||
// systemBackwardTranslator is used to convert headers looking like '__PREFIX__ATTR_NAME' to 'Prefix-Attr-Name'.
|
||||
func (t systemTransformer) systemBackwardTranslator(key string) string {
|
||||
// trim specified prefix '__PREFIX__'
|
||||
key = strings.TrimPrefix(key, t.prefix)
|
||||
|
||||
var res strings.Builder
|
||||
res.WriteString(t.backwardPrefix)
|
||||
|
||||
strs := strings.Split(key, "_")
|
||||
for i, s := range strs {
|
||||
s = title(strings.ToLower(s))
|
||||
res.WriteString(s)
|
||||
if i != len(strs)-1 {
|
||||
res.WriteString("-")
|
||||
}
|
||||
}
|
||||
|
||||
return res.String()
|
||||
}
|
||||
|
||||
func (t systemTransformer) backwardTransformIfSystem(key string) (string, bool) {
|
||||
if strings.HasPrefix(key, t.prefix) {
|
||||
return t.systemBackwardTranslator(key), true
|
||||
}
|
||||
|
||||
return key, false
|
||||
}
|
||||
|
||||
func TransformIfSystem(key []byte) []byte {
|
||||
for _, transformer := range transformers {
|
||||
key, transformed := transformer.transformIfSystem(key)
|
||||
if transformed {
|
||||
return key
|
||||
}
|
||||
}
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func BackwardTransformIfSystem(key string) string {
|
||||
for _, transformer := range transformers {
|
||||
key, transformed := transformer.backwardTransformIfSystem(key)
|
||||
if transformed {
|
||||
return key
|
||||
}
|
||||
}
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func title(str string) string {
|
||||
if str == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
r, size := utf8.DecodeRuneInString(str)
|
||||
r0 := unicode.ToTitle(r)
|
||||
return string(r0) + str[size:]
|
||||
}
|
||||
|
||||
func PrepareExpirationHeader(ctx context.Context, p *pool.Pool, headers map[string]string, now time.Time) error {
|
||||
formatsNum := 0
|
||||
index := -1
|
||||
for i, transformer := range transformers {
|
||||
if transformer.existsExpirationAttributes(headers) {
|
||||
formatsNum++
|
||||
index = i
|
||||
}
|
||||
}
|
||||
|
||||
switch formatsNum {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
epochDuration, err := GetEpochDurations(ctx, p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't get epoch durations from network info: %w", err)
|
||||
}
|
||||
return transformers[index].prepareExpirationHeader(headers, epochDuration, now)
|
||||
default:
|
||||
return errors.New("both deprecated and new system attributes formats are used, please use only one")
|
||||
}
|
||||
}
|
||||
|
||||
func (t systemTransformer) prepareExpirationHeader(headers map[string]string, epochDurations *EpochDurations, now time.Time) error {
|
||||
expirationInEpoch := headers[t.expirationEpochAttr()]
|
||||
|
||||
if timeRFC3339, ok := headers[t.expirationRFC3339Attr()]; ok {
|
||||
expTime, err := time.Parse(time.RFC3339, timeRFC3339)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, t.expirationRFC3339Attr())
|
||||
}
|
||||
|
||||
if expTime.Before(now) {
|
||||
return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, t.expirationRFC3339Attr())
|
||||
}
|
||||
t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
|
||||
delete(headers, t.expirationRFC3339Attr())
|
||||
}
|
||||
|
||||
if timestamp, ok := headers[t.expirationTimestampAttr()]; ok {
|
||||
value, err := strconv.ParseInt(timestamp, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't parse value %s of header %s", timestamp, t.expirationTimestampAttr())
|
||||
}
|
||||
expTime := time.Unix(value, 0)
|
||||
|
||||
if expTime.Before(now) {
|
||||
return fmt.Errorf("value %s of header %s must be in the future", timestamp, t.expirationTimestampAttr())
|
||||
}
|
||||
t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
|
||||
delete(headers, t.expirationTimestampAttr())
|
||||
}
|
||||
|
||||
if duration, ok := headers[t.expirationDurationAttr()]; ok {
|
||||
expDuration, err := time.ParseDuration(duration)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't parse value %s of header %s", duration, t.expirationDurationAttr())
|
||||
}
|
||||
if expDuration <= 0 {
|
||||
return fmt.Errorf("value %s of header %s must be positive", expDuration, t.expirationDurationAttr())
|
||||
}
|
||||
t.updateExpirationHeader(headers, epochDurations, expDuration)
|
||||
delete(headers, t.expirationDurationAttr())
|
||||
}
|
||||
|
||||
if expirationInEpoch != "" {
|
||||
expEpoch, err := strconv.ParseUint(expirationInEpoch, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse expiration epoch '%s': %w", expirationInEpoch, err)
|
||||
}
|
||||
if expEpoch < epochDurations.CurrentEpoch {
|
||||
return fmt.Errorf("expiration epoch '%d' must be greater than current epoch '%d'", expEpoch, epochDurations.CurrentEpoch)
|
||||
}
|
||||
|
||||
headers[t.expirationEpochAttr()] = expirationInEpoch
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t systemTransformer) updateExpirationHeader(headers map[string]string, durations *EpochDurations, expDuration time.Duration) {
|
||||
epochDuration := uint64(durations.MsPerBlock) * durations.BlockPerEpoch
|
||||
currentEpoch := durations.CurrentEpoch
|
||||
numEpoch := uint64(expDuration.Milliseconds()) / epochDuration
|
||||
|
||||
if uint64(expDuration.Milliseconds())%epochDuration != 0 {
|
||||
numEpoch++
|
||||
}
|
||||
|
||||
expirationEpoch := uint64(math.MaxUint64)
|
||||
if numEpoch < math.MaxUint64-currentEpoch {
|
||||
expirationEpoch = currentEpoch + numEpoch
|
||||
}
|
||||
|
||||
headers[t.expirationEpochAttr()] = strconv.FormatUint(expirationEpoch, 10)
|
||||
}
|
||||
|
|
|
|||
187
utils/attributes_test.go
Normal file
187
utils/attributes_test.go
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPrepareExpirationHeader(t *testing.T) {
|
||||
tomorrow := time.Now().Add(24 * time.Hour)
|
||||
tomorrowUnix := tomorrow.Unix()
|
||||
tomorrowUnixNano := tomorrow.UnixNano()
|
||||
tomorrowUnixMilli := tomorrowUnixNano / 1e6
|
||||
|
||||
epoch := "100"
|
||||
duration := "24h"
|
||||
timestampSec := strconv.FormatInt(tomorrowUnix, 10)
|
||||
timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10)
|
||||
timestampNano := strconv.FormatInt(tomorrowUnixNano, 10)
|
||||
|
||||
defaultDurations := &EpochDurations{
|
||||
CurrentEpoch: 10,
|
||||
MsPerBlock: 1000,
|
||||
BlockPerEpoch: 101,
|
||||
}
|
||||
|
||||
msPerBlock := defaultDurations.BlockPerEpoch * uint64(defaultDurations.MsPerBlock)
|
||||
epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock
|
||||
if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 {
|
||||
epochPerDay++
|
||||
}
|
||||
|
||||
defaultExpEpoch := strconv.FormatUint(defaultDurations.CurrentEpoch+epochPerDay, 10)
|
||||
|
||||
for _, transformer := range transformers {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
headers map[string]string
|
||||
durations *EpochDurations
|
||||
err bool
|
||||
expected map[string]string
|
||||
}{
|
||||
{
|
||||
name: "valid epoch",
|
||||
headers: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
durations: defaultDurations,
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid duration",
|
||||
headers: map[string]string{
|
||||
transformer.expirationEpochAttr(): epoch,
|
||||
transformer.expirationDurationAttr(): duration,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid rfc3339",
|
||||
headers: map[string]string{
|
||||
transformer.expirationEpochAttr(): epoch,
|
||||
transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339),
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid timestamp sec",
|
||||
headers: map[string]string{
|
||||
transformer.expirationEpochAttr(): epoch,
|
||||
transformer.expirationTimestampAttr(): timestampSec,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid timestamp milli",
|
||||
headers: map[string]string{
|
||||
transformer.expirationEpochAttr(): epoch,
|
||||
transformer.expirationTimestampAttr(): timestampMilli,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid timestamp nano",
|
||||
headers: map[string]string{
|
||||
transformer.expirationEpochAttr(): epoch,
|
||||
transformer.expirationTimestampAttr(): timestampNano,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
},
|
||||
{
|
||||
name: "valid timestamp sec",
|
||||
headers: map[string]string{transformer.expirationTimestampAttr(): timestampSec},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
|
||||
},
|
||||
{
|
||||
name: "valid duration",
|
||||
headers: map[string]string{transformer.expirationDurationAttr(): duration},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
|
||||
},
|
||||
{
|
||||
name: "valid rfc3339",
|
||||
headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
|
||||
},
|
||||
{
|
||||
name: "valid max uint 64",
|
||||
headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)},
|
||||
durations: &EpochDurations{
|
||||
CurrentEpoch: math.MaxUint64 - 1,
|
||||
MsPerBlock: defaultDurations.MsPerBlock,
|
||||
BlockPerEpoch: defaultDurations.BlockPerEpoch,
|
||||
},
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): strconv.FormatUint(uint64(math.MaxUint64), 10)},
|
||||
},
|
||||
{
|
||||
name: "invalid timestamp sec",
|
||||
headers: map[string]string{transformer.expirationTimestampAttr(): "abc"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid timestamp sec zero",
|
||||
headers: map[string]string{transformer.expirationTimestampAttr(): "0"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid duration",
|
||||
headers: map[string]string{transformer.expirationDurationAttr(): "1d"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid duration negative",
|
||||
headers: map[string]string{transformer.expirationDurationAttr(): "-5h"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid rfc3339",
|
||||
headers: map[string]string{transformer.expirationRFC3339Attr(): "abc"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid rfc3339 zero",
|
||||
headers: map[string]string{transformer.expirationRFC3339Attr(): time.RFC3339},
|
||||
err: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := transformer.prepareExpirationHeader(tc.headers, tc.durations, time.Now())
|
||||
if tc.err {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.expected, tc.headers)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSystemBackwardTranslator(t *testing.T) {
|
||||
input := []string{
|
||||
"__SYSTEM__EXPIRATION_EPOCH",
|
||||
"__SYSTEM__RANDOM_ATTR",
|
||||
"__NEOFS__EXPIRATION_EPOCH",
|
||||
"__NEOFS__RANDOM_ATTR",
|
||||
}
|
||||
expected := []string{
|
||||
"System-Expiration-Epoch",
|
||||
"System-Random-Attr",
|
||||
"Neofs-Expiration-Epoch",
|
||||
"Neofs-Random-Attr",
|
||||
}
|
||||
|
||||
for i, str := range input {
|
||||
res := BackwardTransformIfSystem(str)
|
||||
require.Equal(t, expected[i], res)
|
||||
}
|
||||
}
|
||||
|
|
@ -2,9 +2,11 @@ package utils
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
)
|
||||
|
||||
// GetContainerID decode container id, if it's not a valid container id
|
||||
|
|
@ -17,3 +19,27 @@ func GetContainerID(ctx context.Context, containerID string, resolver *resolver.
|
|||
}
|
||||
return cnrID, err
|
||||
}
|
||||
|
||||
type EpochDurations struct {
|
||||
CurrentEpoch uint64
|
||||
MsPerBlock int64
|
||||
BlockPerEpoch uint64
|
||||
}
|
||||
|
||||
func GetEpochDurations(ctx context.Context, p *pool.Pool) (*EpochDurations, error) {
|
||||
networkInfo, err := p.NetworkInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &EpochDurations{
|
||||
CurrentEpoch: networkInfo.CurrentEpoch(),
|
||||
MsPerBlock: networkInfo.MsPerBlock(),
|
||||
BlockPerEpoch: networkInfo.EpochDuration(),
|
||||
}
|
||||
|
||||
if res.BlockPerEpoch == 0 {
|
||||
return nil, fmt.Errorf("EpochDuration is empty")
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue