forked from TrueCloudLab/frostfs-s3-gw
parent
977f176713
commit
37b1baed41
7 changed files with 95 additions and 262 deletions
|
@ -3,11 +3,14 @@ package layer
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ecdsa"
|
"crypto/ecdsa"
|
||||||
|
"crypto/rand"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
@ -541,10 +544,30 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.Obje
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getRandomOID() (*oid.ID, error) {
|
||||||
|
b := [32]byte{}
|
||||||
|
if _, err := rand.Read(b[:]); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var objID oid.ID
|
||||||
|
objID.SetSHA256(b)
|
||||||
|
return &objID, nil
|
||||||
|
}
|
||||||
|
|
||||||
// DeleteObject removes all objects with the passed nice name.
|
// DeleteObject removes all objects with the passed nice name.
|
||||||
func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings *data.BucketSettings, obj *VersionedObject) *VersionedObject {
|
func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, _ *data.BucketSettings, obj *VersionedObject) *VersionedObject {
|
||||||
if !isRegularVersion(obj.VersionID) {
|
if !isRegularVersion(obj.VersionID) { // version null or empty
|
||||||
|
randOID, err := getRandomOID()
|
||||||
|
if err != nil {
|
||||||
|
obj.Error = fmt.Errorf("couldn't get random oid: %w", err)
|
||||||
|
return obj
|
||||||
|
}
|
||||||
|
obj.DeleteMarkVersion = randOID.EncodeToString()
|
||||||
newVersion := &NodeVersion{
|
newVersion := &NodeVersion{
|
||||||
|
BaseNodeVersion: BaseNodeVersion{
|
||||||
|
OID: *randOID,
|
||||||
|
},
|
||||||
IsDeleteMarker: true,
|
IsDeleteMarker: true,
|
||||||
IsUnversioned: obj.VersionID == unversionedObjectVersionID,
|
IsUnversioned: obj.VersionID == unversionedObjectVersionID,
|
||||||
}
|
}
|
||||||
|
|
|
@ -268,59 +268,6 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func updateCRDT2PSetHeaders(header map[string]string, versions *objectVersions, versioningEnabled bool) []oid.ID {
|
|
||||||
if !versioningEnabled {
|
|
||||||
header[versionsUnversionedAttr] = "true"
|
|
||||||
}
|
|
||||||
|
|
||||||
var idsToDeleteArr []oid.ID
|
|
||||||
if versions.isEmpty() {
|
|
||||||
return idsToDeleteArr
|
|
||||||
}
|
|
||||||
|
|
||||||
if !versions.isAddListEmpty() {
|
|
||||||
header[versionsAddAttr] = versions.getAddHeader()
|
|
||||||
}
|
|
||||||
|
|
||||||
if versioningEnabled {
|
|
||||||
versionsDeletedStr := versions.getDelHeader()
|
|
||||||
// header[versionsDelAttr] can be not empty when deleting specific version
|
|
||||||
if delAttr := header[versionsDelAttr]; len(delAttr) != 0 {
|
|
||||||
if len(versionsDeletedStr) != 0 {
|
|
||||||
header[versionsDelAttr] = versionsDeletedStr + "," + delAttr
|
|
||||||
} else {
|
|
||||||
header[versionsDelAttr] = delAttr
|
|
||||||
}
|
|
||||||
} else if len(versionsDeletedStr) != 0 {
|
|
||||||
header[versionsDelAttr] = versionsDeletedStr
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
versionsDeletedStr := versions.getDelHeader()
|
|
||||||
|
|
||||||
var additionalDel string
|
|
||||||
for i, del := range versions.unversioned() {
|
|
||||||
if i != 0 {
|
|
||||||
additionalDel += ","
|
|
||||||
}
|
|
||||||
additionalDel += del.Version()
|
|
||||||
idsToDeleteArr = append(idsToDeleteArr, del.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(additionalDel) != 0 {
|
|
||||||
if len(versionsDeletedStr) != 0 {
|
|
||||||
versionsDeletedStr += ","
|
|
||||||
}
|
|
||||||
versionsDeletedStr += additionalDel
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(versionsDeletedStr) != 0 {
|
|
||||||
header[versionsDelAttr] = versionsDeletedStr
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return idsToDeleteArr
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ObjectInfo, error) {
|
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ObjectInfo, error) {
|
||||||
if addr := n.namesCache.Get(bkt.Name + "/" + objectName); addr != nil {
|
if addr := n.namesCache.Get(bkt.Name + "/" + objectName); addr != nil {
|
||||||
if headInfo := n.objCache.Get(*addr); headInfo != nil {
|
if headInfo := n.objCache.Get(*addr); headInfo != nil {
|
||||||
|
@ -330,6 +277,9 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
|
||||||
|
|
||||||
node, err := n.treeService.GetLatestVersion(ctx, &bkt.CID, objectName)
|
node, err := n.treeService.GetLatestVersion(ctx, &bkt.CID, objectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrNodeNotFound) {
|
||||||
|
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
|
||||||
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -410,16 +360,28 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
|
||||||
return objInfo, nil
|
return objInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var id oid.ID
|
versions, err := n.treeService.GetVersions(ctx, &bkt.CID, p.Object)
|
||||||
if err := id.DecodeString(p.VersionID); err != nil {
|
if err != nil {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidVersion)
|
return nil, fmt.Errorf("couldn't get versions: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if headInfo := n.objCache.Get(newAddress(bkt.CID, id)); headInfo != nil {
|
var foundVersion *NodeVersion
|
||||||
|
for _, version := range versions {
|
||||||
|
if version.OID.EncodeToString() == p.VersionID {
|
||||||
|
foundVersion = version
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if foundVersion == nil {
|
||||||
|
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
if headInfo := n.objCache.Get(newAddress(bkt.CID, foundVersion.OID)); headInfo != nil {
|
||||||
return objInfoFromMeta(bkt, headInfo), nil
|
return objInfoFromMeta(bkt, headInfo), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
meta, err := n.objectHead(ctx, bkt, id)
|
meta, err := n.objectHead(ctx, bkt, foundVersion.OID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if client.IsErrObjectNotFound(err) {
|
if client.IsErrObjectNotFound(err) {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion)
|
||||||
|
|
|
@ -44,6 +44,9 @@ func (n *layer) HeadSystemObject(ctx context.Context, bkt *data.BucketInfo, objN
|
||||||
|
|
||||||
node, err := n.treeService.GetSystemVersion(ctx, &bkt.CID, objName)
|
node, err := n.treeService.GetSystemVersion(ctx, &bkt.CID, objName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errorsStd.Is(err, ErrNodeNotFound) {
|
||||||
|
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
|
||||||
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -31,19 +31,13 @@ type TreeService interface {
|
||||||
DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid.ID, error)
|
DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid.ID, error)
|
||||||
|
|
||||||
GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]*NodeVersion, error)
|
GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]*NodeVersion, error)
|
||||||
|
|
||||||
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*NodeVersion, error)
|
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*NodeVersion, error)
|
||||||
|
|
||||||
GetUnversioned(ctx context.Context, cnrID *cid.ID, objectName string) (*NodeVersion, error)
|
GetUnversioned(ctx context.Context, cnrID *cid.ID, objectName string) (*NodeVersion, error)
|
||||||
|
|
||||||
AddVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *NodeVersion) error
|
AddVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *NodeVersion) error
|
||||||
|
|
||||||
RemoveVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error
|
RemoveVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error
|
||||||
|
|
||||||
AddSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *BaseNodeVersion) error
|
AddSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *BaseNodeVersion) error
|
||||||
|
|
||||||
GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*BaseNodeVersion, error)
|
GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*BaseNodeVersion, error)
|
||||||
|
|
||||||
RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error
|
RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -8,8 +8,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type objectVersions struct {
|
type objectVersions struct {
|
||||||
|
@ -57,11 +55,6 @@ func newObjectVersions(name string) *objectVersions {
|
||||||
return &objectVersions{name: name}
|
return &objectVersions{name: name}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *objectVersions) isAddListEmpty() bool {
|
|
||||||
v.sort()
|
|
||||||
return len(v.addList) == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *objectVersions) appendVersion(oi *data.ObjectInfo) {
|
func (v *objectVersions) appendVersion(oi *data.ObjectInfo) {
|
||||||
delVers := splitVersions(oi.Headers[versionsDelAttr])
|
delVers := splitVersions(oi.Headers[versionsDelAttr])
|
||||||
v.objects = append(v.objects, oi)
|
v.objects = append(v.objects, oi)
|
||||||
|
@ -182,23 +175,6 @@ func (v *objectVersions) isEmpty() bool {
|
||||||
return v == nil || len(v.objects) == 0
|
return v == nil || len(v.objects) == 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *objectVersions) unversioned() []*data.ObjectInfo {
|
|
||||||
if len(v.objects) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
existedVersions := v.existedVersions()
|
|
||||||
res := make([]*data.ObjectInfo, 0, len(v.objects))
|
|
||||||
|
|
||||||
for _, version := range v.objects {
|
|
||||||
if contains(existedVersions, version.Version()) && version.Headers[versionsUnversionedAttr] == "true" {
|
|
||||||
res = append(res, version)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *objectVersions) getLast(opts ...VersionOption) *data.ObjectInfo {
|
func (v *objectVersions) getLast(opts ...VersionOption) *data.ObjectInfo {
|
||||||
if v.isEmpty() {
|
if v.isEmpty() {
|
||||||
return nil
|
return nil
|
||||||
|
@ -267,22 +243,6 @@ func (v *objectVersions) getAddHeader() string {
|
||||||
return strings.Join(v.addList, ",")
|
return strings.Join(v.addList, ",")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *objectVersions) getDelHeader() string {
|
|
||||||
return strings.Join(v.delList, ",")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *objectVersions) getVersion(obj oid.ID) *data.ObjectInfo {
|
|
||||||
strObj := obj.EncodeToString()
|
|
||||||
for _, version := range v.objects {
|
|
||||||
if version.Version() == strObj {
|
|
||||||
if contains(v.delList, strObj) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return version
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (n *layer) PutBucketVersioning(ctx context.Context, p *PutSettingsParams) (*data.ObjectInfo, error) {
|
func (n *layer) PutBucketVersioning(ctx context.Context, p *PutSettingsParams) (*data.ObjectInfo, error) {
|
||||||
metadata := map[string]string{
|
metadata := map[string]string{
|
||||||
attrSettingsVersioningEnabled: strconv.FormatBool(p.Settings.VersioningEnabled),
|
attrSettingsVersioningEnabled: strconv.FormatBool(p.Settings.VersioningEnabled),
|
||||||
|
@ -383,27 +343,3 @@ func contains(list []string, elem string) bool {
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) checkVersionsExist(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.ObjectInfo, error) {
|
|
||||||
versions, err := n.headVersions(ctx, bkt, obj.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var version *data.ObjectInfo
|
|
||||||
if obj.VersionID == unversionedObjectVersionID {
|
|
||||||
version = versions.getLast(FromUnversioned())
|
|
||||||
} else {
|
|
||||||
var id oid.ID
|
|
||||||
if err = id.DecodeString(obj.VersionID); err != nil {
|
|
||||||
return nil, errors.GetAPIError(errors.ErrInvalidVersion)
|
|
||||||
}
|
|
||||||
version = versions.getVersion(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
if version == nil {
|
|
||||||
return nil, errors.GetAPIError(errors.ErrInvalidVersion)
|
|
||||||
}
|
|
||||||
|
|
||||||
return version, nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -506,108 +506,12 @@ func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *data.Obje
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getTestUnversionedObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo {
|
|
||||||
objInfo := getTestObjectInfo(id, addAttr, delAttr, delMarkAttr)
|
|
||||||
objInfo.Headers[versionsUnversionedAttr] = "true"
|
|
||||||
return objInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
func getTestObjectInfoEpoch(epoch uint64, id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo {
|
func getTestObjectInfoEpoch(epoch uint64, id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo {
|
||||||
obj := getTestObjectInfo(id, addAttr, delAttr, delMarkAttr)
|
obj := getTestObjectInfo(id, addAttr, delAttr, delMarkAttr)
|
||||||
obj.CreationEpoch = epoch
|
obj.CreationEpoch = epoch
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUpdateCRDT2PSetHeaders(t *testing.T) {
|
|
||||||
obj1 := getTestUnversionedObjectInfo(1, "", "", "")
|
|
||||||
obj2 := getTestUnversionedObjectInfo(2, "", "", "")
|
|
||||||
obj3 := getTestObjectInfo(3, "", "", "")
|
|
||||||
obj4 := getTestObjectInfo(4, "", "", "")
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
header map[string]string
|
|
||||||
versions *objectVersions
|
|
||||||
versioningEnabled bool
|
|
||||||
expectedHeader map[string]string
|
|
||||||
expectedIdsToDelete []oid.ID
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "unversioned save headers",
|
|
||||||
header: map[string]string{"someKey": "someValue"},
|
|
||||||
expectedHeader: map[string]string{"someKey": "someValue", versionsUnversionedAttr: "true"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unversioned put",
|
|
||||||
header: map[string]string{},
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj1},
|
|
||||||
},
|
|
||||||
expectedHeader: map[string]string{
|
|
||||||
versionsAddAttr: obj1.Version(),
|
|
||||||
versionsDelAttr: obj1.Version(),
|
|
||||||
versionsUnversionedAttr: "true",
|
|
||||||
},
|
|
||||||
expectedIdsToDelete: []oid.ID{obj1.ID},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unversioned del header",
|
|
||||||
header: map[string]string{},
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj2},
|
|
||||||
delList: []string{obj1.Version()},
|
|
||||||
},
|
|
||||||
expectedHeader: map[string]string{
|
|
||||||
versionsAddAttr: obj2.Version(),
|
|
||||||
versionsDelAttr: joinVers(obj1, obj2),
|
|
||||||
versionsUnversionedAttr: "true",
|
|
||||||
},
|
|
||||||
expectedIdsToDelete: []oid.ID{obj2.ID},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "versioned put",
|
|
||||||
header: map[string]string{},
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj3},
|
|
||||||
},
|
|
||||||
versioningEnabled: true,
|
|
||||||
expectedHeader: map[string]string{versionsAddAttr: obj3.Version()},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "versioned del header",
|
|
||||||
header: map[string]string{versionsDelAttr: obj4.Version()},
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj4},
|
|
||||||
delList: []string{obj3.Version()},
|
|
||||||
},
|
|
||||||
versioningEnabled: true,
|
|
||||||
expectedHeader: map[string]string{
|
|
||||||
versionsAddAttr: obj4.Version(),
|
|
||||||
versionsDelAttr: joinVers(obj3, obj4),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unversioned put after some version",
|
|
||||||
header: map[string]string{},
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj1, obj3},
|
|
||||||
},
|
|
||||||
expectedHeader: map[string]string{
|
|
||||||
versionsAddAttr: joinVers(obj1, obj3),
|
|
||||||
versionsDelAttr: obj1.Version(),
|
|
||||||
versionsUnversionedAttr: "true",
|
|
||||||
},
|
|
||||||
expectedIdsToDelete: []oid.ID{obj1.ID},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
idsToDelete := updateCRDT2PSetHeaders(tc.header, tc.versions, tc.versioningEnabled)
|
|
||||||
require.Equal(t, tc.expectedHeader, tc.header)
|
|
||||||
require.Equal(t, tc.expectedIdsToDelete, idsToDelete)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSystemObjectsVersioning(t *testing.T) {
|
func TestSystemObjectsVersioning(t *testing.T) {
|
||||||
cacheConfig := DefaultCachesConfigs(zap.NewExample())
|
cacheConfig := DefaultCachesConfigs(zap.NewExample())
|
||||||
cacheConfig.System.Lifetime = 0
|
cacheConfig.System.Lifetime = 0
|
||||||
|
|
|
@ -7,8 +7,10 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/internal/neofs/services/tree"
|
"github.com/nspcc-dev/neofs-s3-gw/internal/neofs/services/tree"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||||
|
@ -24,7 +26,7 @@ type (
|
||||||
|
|
||||||
TreeNode struct {
|
TreeNode struct {
|
||||||
ID uint64
|
ID uint64
|
||||||
ObjID *oid.ID
|
ObjID oid.ID
|
||||||
TimeStamp uint64
|
TimeStamp uint64
|
||||||
Meta map[string]string
|
Meta map[string]string
|
||||||
}
|
}
|
||||||
|
@ -43,12 +45,12 @@ const (
|
||||||
notifConfFileName = "bucket-notifications"
|
notifConfFileName = "bucket-notifications"
|
||||||
corsFilename = "bucket-cors"
|
corsFilename = "bucket-cors"
|
||||||
|
|
||||||
// bucketSystemObjectsTreeID -- ID of a tree with system objects for bucket
|
// versionTree -- ID of a tree with object versions.
|
||||||
// i.e. bucket settings with versioning and lock configuration, cors, notifications
|
|
||||||
bucketSystemObjectsTreeID = "system-bucket"
|
|
||||||
|
|
||||||
versionTree = "version"
|
versionTree = "version"
|
||||||
systemTree = "system"
|
|
||||||
|
// systemTree -- ID of a tree with system objects
|
||||||
|
// i.e. bucket settings with versioning and lock configuration, cors, notifications.
|
||||||
|
systemTree = "system"
|
||||||
|
|
||||||
separator = "/"
|
separator = "/"
|
||||||
)
|
)
|
||||||
|
@ -69,14 +71,12 @@ func NewTreeClient(addr string) (*TreeClient, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTreeNode(nodeInfo *tree.GetNodeByPathResponse_Info) (*TreeNode, error) {
|
func newTreeNode(nodeInfo *tree.GetNodeByPathResponse_Info) (*TreeNode, error) {
|
||||||
var objID *oid.ID
|
var objID oid.ID
|
||||||
meta := make(map[string]string, len(nodeInfo.GetMeta()))
|
meta := make(map[string]string, len(nodeInfo.GetMeta()))
|
||||||
|
|
||||||
for _, kv := range nodeInfo.GetMeta() {
|
for _, kv := range nodeInfo.GetMeta() {
|
||||||
if kv.GetKey() == oidKV {
|
if kv.GetKey() == oidKV {
|
||||||
objID = new(oid.ID)
|
if err := objID.DecodeString(string(kv.GetValue())); err != nil {
|
||||||
err := objID.DecodeString(string(kv.GetValue()))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
|
@ -110,7 +110,7 @@ func newNodeVersion(node *tree.GetNodeByPathResponse_Info) (*layer.NodeVersion,
|
||||||
return &layer.NodeVersion{
|
return &layer.NodeVersion{
|
||||||
BaseNodeVersion: layer.BaseNodeVersion{
|
BaseNodeVersion: layer.BaseNodeVersion{
|
||||||
ID: node.NodeId,
|
ID: node.NodeId,
|
||||||
OID: *treeNode.ObjID,
|
OID: treeNode.ObjID,
|
||||||
},
|
},
|
||||||
IsUnversioned: isUnversioned,
|
IsUnversioned: isUnversioned,
|
||||||
IsDeleteMarker: isDeleteMarker,
|
IsDeleteMarker: isDeleteMarker,
|
||||||
|
@ -119,8 +119,7 @@ func newNodeVersion(node *tree.GetNodeByPathResponse_Info) (*layer.NodeVersion,
|
||||||
|
|
||||||
func (c *TreeClient) GetSettingsNode(ctx context.Context, cnrID *cid.ID) (*data.BucketSettings, error) {
|
func (c *TreeClient) GetSettingsNode(ctx context.Context, cnrID *cid.ID) (*data.BucketSettings, error) {
|
||||||
keysToReturn := []string{versioningEnabledKV, lockConfigurationKV}
|
keysToReturn := []string{versioningEnabledKV, lockConfigurationKV}
|
||||||
path := []string{settingsFileName}
|
node, err := c.getSystemNode(ctx, cnrID, systemTree, []string{settingsFileName}, keysToReturn)
|
||||||
node, err := c.getSystemNode(ctx, cnrID, bucketSystemObjectsTreeID, path, keysToReturn)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("couldn't get node: %w", err)
|
return nil, fmt.Errorf("couldn't get node: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -143,8 +142,7 @@ func (c *TreeClient) GetSettingsNode(ctx context.Context, cnrID *cid.ID) (*data.
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *TreeClient) PutSettingsNode(ctx context.Context, cnrID *cid.ID, settings *data.BucketSettings) error {
|
func (c *TreeClient) PutSettingsNode(ctx context.Context, cnrID *cid.ID, settings *data.BucketSettings) error {
|
||||||
path := []string{settingsFileName}
|
node, err := c.getSystemNode(ctx, cnrID, systemTree, []string{settingsFileName}, []string{})
|
||||||
node, err := c.getSystemNode(ctx, cnrID, bucketSystemObjectsTreeID, path, []string{})
|
|
||||||
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
||||||
if err != nil && !isErrNotFound {
|
if err != nil && !isErrNotFound {
|
||||||
return fmt.Errorf("couldn't get node: %w", err)
|
return fmt.Errorf("couldn't get node: %w", err)
|
||||||
|
@ -153,26 +151,24 @@ func (c *TreeClient) PutSettingsNode(ctx context.Context, cnrID *cid.ID, setting
|
||||||
meta := metaFromSettings(settings)
|
meta := metaFromSettings(settings)
|
||||||
|
|
||||||
if isErrNotFound {
|
if isErrNotFound {
|
||||||
_, err = c.addNode(ctx, cnrID, bucketSystemObjectsTreeID, 0, meta)
|
_, err = c.addNode(ctx, cnrID, systemTree, 0, meta)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.moveNode(ctx, cnrID, bucketSystemObjectsTreeID, node.ID, 0, meta)
|
return c.moveNode(ctx, cnrID, systemTree, node.ID, 0, meta)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *TreeClient) GetNotificationConfigurationNode(ctx context.Context, cnrID *cid.ID) (*oid.ID, error) {
|
func (c *TreeClient) GetNotificationConfigurationNode(ctx context.Context, cnrID *cid.ID) (*oid.ID, error) {
|
||||||
path := []string{notifConfFileName}
|
node, err := c.getSystemNode(ctx, cnrID, systemTree, []string{notifConfFileName}, []string{oidKV})
|
||||||
node, err := c.getSystemNode(ctx, cnrID, bucketSystemObjectsTreeID, path, []string{oidKV})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return node.ObjID, nil
|
return &node.ObjID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *TreeClient) PutNotificationConfigurationNode(ctx context.Context, cnrID *cid.ID, objID *oid.ID) (*oid.ID, error) {
|
func (c *TreeClient) PutNotificationConfigurationNode(ctx context.Context, cnrID *cid.ID, objID *oid.ID) (*oid.ID, error) {
|
||||||
path := []string{notifConfFileName}
|
node, err := c.getSystemNode(ctx, cnrID, systemTree, []string{notifConfFileName}, []string{oidKV})
|
||||||
node, err := c.getSystemNode(ctx, cnrID, bucketSystemObjectsTreeID, path, []string{oidKV})
|
|
||||||
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
||||||
if err != nil && !isErrNotFound {
|
if err != nil && !isErrNotFound {
|
||||||
return nil, fmt.Errorf("couldn't get node: %w", err)
|
return nil, fmt.Errorf("couldn't get node: %w", err)
|
||||||
|
@ -183,24 +179,24 @@ func (c *TreeClient) PutNotificationConfigurationNode(ctx context.Context, cnrID
|
||||||
meta[oidKV] = objID.EncodeToString()
|
meta[oidKV] = objID.EncodeToString()
|
||||||
|
|
||||||
if isErrNotFound {
|
if isErrNotFound {
|
||||||
_, err = c.addNode(ctx, cnrID, bucketSystemObjectsTreeID, 0, meta)
|
_, err = c.addNode(ctx, cnrID, systemTree, 0, meta)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return node.ObjID, c.moveNode(ctx, cnrID, bucketSystemObjectsTreeID, node.ID, 0, meta)
|
return &node.ObjID, c.moveNode(ctx, cnrID, systemTree, node.ID, 0, meta)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *TreeClient) GetBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid.ID, error) {
|
func (c *TreeClient) GetBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid.ID, error) {
|
||||||
node, err := c.getSystemNode(ctx, cnrID, bucketSystemObjectsTreeID, []string{corsFilename}, []string{oidKV})
|
node, err := c.getSystemNode(ctx, cnrID, systemTree, []string{corsFilename}, []string{oidKV})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return node.ObjID, nil
|
return &node.ObjID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *TreeClient) PutBucketCORS(ctx context.Context, cnrID *cid.ID, objID *oid.ID) (*oid.ID, error) {
|
func (c *TreeClient) PutBucketCORS(ctx context.Context, cnrID *cid.ID, objID *oid.ID) (*oid.ID, error) {
|
||||||
node, err := c.getSystemNode(ctx, cnrID, bucketSystemObjectsTreeID, []string{corsFilename}, []string{oidKV})
|
node, err := c.getSystemNode(ctx, cnrID, systemTree, []string{corsFilename}, []string{oidKV})
|
||||||
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
||||||
if err != nil && !isErrNotFound {
|
if err != nil && !isErrNotFound {
|
||||||
return nil, fmt.Errorf("couldn't get node: %w", err)
|
return nil, fmt.Errorf("couldn't get node: %w", err)
|
||||||
|
@ -211,21 +207,21 @@ func (c *TreeClient) PutBucketCORS(ctx context.Context, cnrID *cid.ID, objID *oi
|
||||||
meta[oidKV] = objID.EncodeToString()
|
meta[oidKV] = objID.EncodeToString()
|
||||||
|
|
||||||
if isErrNotFound {
|
if isErrNotFound {
|
||||||
_, err = c.addNode(ctx, cnrID, bucketSystemObjectsTreeID, 0, meta)
|
_, err = c.addNode(ctx, cnrID, systemTree, 0, meta)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return node.ObjID, c.moveNode(ctx, cnrID, bucketSystemObjectsTreeID, node.ID, 0, meta)
|
return &node.ObjID, c.moveNode(ctx, cnrID, systemTree, node.ID, 0, meta)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *TreeClient) DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid.ID, error) {
|
func (c *TreeClient) DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid.ID, error) {
|
||||||
node, err := c.getSystemNode(ctx, cnrID, bucketSystemObjectsTreeID, []string{corsFilename}, []string{oidKV})
|
node, err := c.getSystemNode(ctx, cnrID, systemTree, []string{corsFilename}, []string{oidKV})
|
||||||
if err != nil && !errors.Is(err, layer.ErrNodeNotFound) {
|
if err != nil && !errors.Is(err, layer.ErrNodeNotFound) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if node != nil {
|
if node != nil {
|
||||||
return node.ObjID, c.removeNode(ctx, cnrID, bucketSystemObjectsTreeID, node.ID)
|
return &node.ObjID, c.removeNode(ctx, cnrID, systemTree, node.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
@ -262,6 +258,10 @@ func (c *TreeClient) getLatestVersion(ctx context.Context, cnrID *cid.ID, treeID
|
||||||
return nil, fmt.Errorf("couldn't get nodes: %w", err)
|
return nil, fmt.Errorf("couldn't get nodes: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return nil, layer.ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
return newNodeVersion(nodes[0])
|
return newNodeVersion(nodes[0])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -352,6 +352,7 @@ func (c *TreeClient) removeVersion(ctx context.Context, cnrID *cid.ID, treeID st
|
||||||
ContainerId: []byte(cnrID.EncodeToString()),
|
ContainerId: []byte(cnrID.EncodeToString()),
|
||||||
TreeId: treeID,
|
TreeId: treeID,
|
||||||
NodeId: id,
|
NodeId: id,
|
||||||
|
BearerToken: getBearer(ctx),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -360,7 +361,7 @@ func (c *TreeClient) removeVersion(ctx context.Context, cnrID *cid.ID, treeID st
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *TreeClient) getVersions(ctx context.Context, cnrID *cid.ID, treeID, filepath string, onlyUnversioned bool) ([]*layer.NodeVersion, error) {
|
func (c *TreeClient) getVersions(ctx context.Context, cnrID *cid.ID, treeID, filepath string, onlyUnversioned bool) ([]*layer.NodeVersion, error) {
|
||||||
keysToReturn := []string{versioningEnabledKV, lockConfigurationKV}
|
keysToReturn := []string{oidKV, isUnversionedKV, isDeleteMarkerKV}
|
||||||
path := strings.Split(filepath, separator)
|
path := strings.Split(filepath, separator)
|
||||||
nodes, err := c.getNodes(ctx, cnrID, treeID, fileNameKV, path, keysToReturn, false)
|
nodes, err := c.getNodes(ctx, cnrID, treeID, fileNameKV, path, keysToReturn, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -393,6 +394,7 @@ func (c *TreeClient) getParent(ctx context.Context, cnrID *cid.ID, treeID string
|
||||||
ContainerId: []byte(cnrID.EncodeToString()),
|
ContainerId: []byte(cnrID.EncodeToString()),
|
||||||
TreeId: treeID,
|
TreeId: treeID,
|
||||||
RootId: id,
|
RootId: id,
|
||||||
|
BearerToken: getBearer(ctx),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -423,10 +425,6 @@ func (c *TreeClient) getSystemNode(ctx context.Context, cnrID *cid.ID, treeID st
|
||||||
return c.getNode(ctx, cnrID, treeID, systemNameKV, path, meta)
|
return c.getNode(ctx, cnrID, treeID, systemNameKV, path, meta)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *TreeClient) getRegularNode(ctx context.Context, cnrID *cid.ID, treeID string, path, meta []string) (*TreeNode, error) {
|
|
||||||
return c.getNode(ctx, cnrID, treeID, fileNameKV, path, meta)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *TreeClient) getNode(ctx context.Context, cnrID *cid.ID, treeID, pathAttr string, path, meta []string) (*TreeNode, error) {
|
func (c *TreeClient) getNode(ctx context.Context, cnrID *cid.ID, treeID, pathAttr string, path, meta []string) (*TreeNode, error) {
|
||||||
nodes, err := c.getNodes(ctx, cnrID, treeID, pathAttr, path, meta, false)
|
nodes, err := c.getNodes(ctx, cnrID, treeID, pathAttr, path, meta, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -454,6 +452,7 @@ func (c *TreeClient) getNodes(ctx context.Context, cnrID *cid.ID, treeID, pathAt
|
||||||
Attributes: meta,
|
Attributes: meta,
|
||||||
PathAttribute: pathAttr,
|
PathAttribute: pathAttr,
|
||||||
LatestOnly: latestOnly,
|
LatestOnly: latestOnly,
|
||||||
|
BearerToken: getBearer(ctx),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -465,6 +464,15 @@ func (c *TreeClient) getNodes(ctx context.Context, cnrID *cid.ID, treeID, pathAt
|
||||||
return resp.GetBody().GetNodes(), nil
|
return resp.GetBody().GetNodes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getBearer(ctx context.Context) []byte {
|
||||||
|
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil {
|
||||||
|
if bd.Gate.BearerToken != nil {
|
||||||
|
return bd.Gate.BearerToken.Marshal()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *TreeClient) addNode(ctx context.Context, cnrID *cid.ID, treeID string, parent uint64, meta map[string]string) (uint64, error) {
|
func (c *TreeClient) addNode(ctx context.Context, cnrID *cid.ID, treeID string, parent uint64, meta map[string]string) (uint64, error) {
|
||||||
request := &tree.AddRequest{
|
request := &tree.AddRequest{
|
||||||
Body: &tree.AddRequest_Body{
|
Body: &tree.AddRequest_Body{
|
||||||
|
@ -472,6 +480,7 @@ func (c *TreeClient) addNode(ctx context.Context, cnrID *cid.ID, treeID string,
|
||||||
TreeId: treeID,
|
TreeId: treeID,
|
||||||
ParentId: parent,
|
ParentId: parent,
|
||||||
Meta: metaToKV(meta),
|
Meta: metaToKV(meta),
|
||||||
|
BearerToken: getBearer(ctx),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -491,6 +500,7 @@ func (c *TreeClient) addNodeByPath(ctx context.Context, cnrID *cid.ID, treeID st
|
||||||
Path: path,
|
Path: path,
|
||||||
Meta: metaToKV(meta),
|
Meta: metaToKV(meta),
|
||||||
PathAttribute: fileNameKV,
|
PathAttribute: fileNameKV,
|
||||||
|
BearerToken: getBearer(ctx),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -519,6 +529,7 @@ func (c *TreeClient) removeNode(ctx context.Context, cnrID *cid.ID, treeID strin
|
||||||
ContainerId: []byte(cnrID.EncodeToString()),
|
ContainerId: []byte(cnrID.EncodeToString()),
|
||||||
TreeId: treeID,
|
TreeId: treeID,
|
||||||
NodeId: nodeID,
|
NodeId: nodeID,
|
||||||
|
BearerToken: getBearer(ctx),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
_, err := c.service.Remove(ctx, r)
|
_, err := c.service.Remove(ctx, r)
|
||||||
|
|
Loading…
Reference in a new issue