pogpp/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Pull requests Activity

[#590] Use service records to save resource info

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-07-15 17:35:11 +03:00 committed by Alex Vanin
parent b144e50f7f
commit 1e26cf1541
2 changed files with 230 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

258
api/handler/acl.go
View file

@ -10,6 +10,8 @@ import (
stderrors "errors"
"fmt"
"net/http"
"sort"
"strconv"
"strings"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
@ -92,6 +94,11 @@ type principal struct {
CanonicalUser string `json:"CanonicalUser,omitempty"`
}
type orderedAstResource struct {
Index int
Resource *astResource
}
type ast struct {
Resources []*astResource
}
@ -131,6 +138,23 @@ func (a astOperation) IsGroupGrantee() bool {
return len(a.Users) == 0
}
const (
serviceRecordResourceKey = "Resource"
serviceRecordGroupLengthKey = "GroupLength"
)
type ServiceRecord struct {
Resource string
GroupRecordsLength int
}
func (s ServiceRecord) ToEACLRecord() *eacl.Record {
serviceRecord := eacl.NewRecord()
serviceRecord.AddFilter(eacl.HeaderFromService, eacl.MatchUnknown, serviceRecordResourceKey, s.Resource)
serviceRecord.AddFilter(eacl.HeaderFromService, eacl.MatchUnknown, serviceRecordGroupLengthKey, strconv.Itoa(s.GroupRecordsLength))
return serviceRecord
}
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
@ -146,7 +170,7 @@ func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
return
}
if err = api.EncodeToResponse(w, h.encodeBucketACL(bucketACL)); err != nil {
if err = api.EncodeToResponse(w, h.encodeBucketACL(bktInfo.Name, bucketACL)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
return
}
@ -268,8 +292,20 @@ func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
return
}
if err = api.EncodeToResponse(w, h.encodeObjectACL(bucketACL, reqInfo.ObjectName)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
prm := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
objInfo, err := h.obj.GetObjectInfo(r.Context(), prm)
if err != nil {
h.logAndSendError(w, "could not object info", reqInfo, err)
return
}
if err = api.EncodeToResponse(w, h.encodeObjectACL(bucketACL, reqInfo.BucketName, objInfo.Version())); err != nil {
h.logAndSendError(w, "failed to encode response", reqInfo, err)
}
}
@ -566,51 +602,87 @@ func addPredefinedACP(acp *AccessControlPolicy, cannedACL string) (*AccessContro
}
func tableToAst(table *eacl.Table, bktName string) *ast {
result := &ast{}
metResources := make(map[string]int)
resourceMap := make(map[string]orderedAstResource)
for i := len(table.Records()) - 1; i >= 0; i-- {
resName := bktName
var objectName string
var version string
record := table.Records()[i]
for _, filter := range record.Filters() {
if filter.Matcher() == eacl.MatchStringEqual {
if filter.Key() == object.AttributeFileName {
objectName = filter.Value()
resName += "/" + objectName
} else if filter.Key() == v2acl.FilterObjectID {
version = filter.Value()
resName += "/" + version
}
}
}
idx, ok := metResources[resName]
if !ok {
resource := &astResource{resourceInfo: resourceInfo{
Bucket: bktName,
Object: objectName,
Version: version,
}}
result.Resources = append(result.Resources, resource)
idx = len(result.Resources) - 1
metResources[resName] = idx
}
var groupRecordsLeft int
var currentResource orderedAstResource
for i, record := range table.Records() {
if serviceRec := tryServiceRecord(record); serviceRec != nil {
resInfo := resourceInfoFromName(serviceRec.Resource, bktName)
groupRecordsLeft = serviceRec.GroupRecordsLength
for _, target := range record.Targets() {
result.Resources[idx].Operations = addToList(result.Resources[idx].Operations, record, target)
currentResource = getResourceOrCreate(resourceMap, i, resInfo)
resourceMap[resInfo.Name()] = currentResource
} else if groupRecordsLeft != 0 {
groupRecordsLeft--
addOperationsAndUpdateMap(currentResource, record, resourceMap)
} else {
resInfo := resInfoFromFilters(bktName, record.Filters())
resource := getResourceOrCreate(resourceMap, i, resInfo)
addOperationsAndUpdateMap(resource, record, resourceMap)
}
}
for _, res := range result.Resources {
for i, j := 0, len(res.Operations)-1; i < j; i, j = i+1, j-1 {
res.Operations[i], res.Operations[j] = res.Operations[j], res.Operations[i]
return &ast{
Resources: formReverseOrderResources(resourceMap),
}
}
func formReverseOrderResources(resourceMap map[string]orderedAstResource) []*astResource {
orderedResources := make([]orderedAstResource, 0, len(resourceMap))
for _, resource := range resourceMap {
orderedResources = append(orderedResources, resource)
}
sort.Slice(orderedResources, func(i, j int) bool {
return orderedResources[i].Index >= orderedResources[j].Index // reverse order
})
result := make([]*astResource, len(orderedResources))
for i, ordered := range orderedResources {
res := ordered.Resource
for j, k := 0, len(res.Operations)-1; j < k; j, k = j+1, k-1 {
res.Operations[j], res.Operations[k] = res.Operations[k], res.Operations[j]
}
result[i] = res
}
return result
}
func addOperationsAndUpdateMap(orderedRes orderedAstResource, record eacl.Record, resMap map[string]orderedAstResource) {
for _, target := range record.Targets() {
orderedRes.Resource.Operations = addToList(orderedRes.Resource.Operations, record, target)
}
resMap[orderedRes.Resource.Name()] = orderedRes
}
func getResourceOrCreate(resMap map[string]orderedAstResource, index int, resInfo resourceInfo) orderedAstResource {
resource, ok := resMap[resInfo.Name()]
if !ok {
resource = orderedAstResource{
Index: index,
Resource: &astResource{resourceInfo: resInfo},
}
}
return resource
}
func resInfoFromFilters(bucketName string, filters []eacl.Filter) resourceInfo {
resInfo := resourceInfo{Bucket: bucketName}
for _, filter := range filters {
if filter.Matcher() == eacl.MatchStringEqual {
if filter.Key() == object.AttributeFileName {
resInfo.Object = filter.Value()
} else if filter.Key() == v2acl.FilterObjectID {
resInfo.Version = filter.Value()
}
}
}
return resInfo
}
func mergeAst(parent, child *ast) (*ast, bool) {
updated := false
for _, resource := range child.Resources {
@ -788,6 +860,13 @@ func astToTable(ast *ast) (*eacl.Table, error) {
if err != nil {
return nil, fmt.Errorf("form records: %w", err)
}
serviceRecord := ServiceRecord{
Resource: ast.Resources[i].Name(),
GroupRecordsLength: len(records),
}
table.AddRecord(serviceRecord.ToEACLRecord())
for _, rec := range records {
table.AddRecord(rec)
}
@ -796,10 +875,36 @@ func astToTable(ast *ast) (*eacl.Table, error) {
return table, nil
}
func tryServiceRecord(record eacl.Record) *ServiceRecord {
if record.Action() != eacl.ActionUnknown || len(record.Targets()) != 0 ||
len(record.Filters()) != 2 {
return nil
}
resourceFilter := record.Filters()[0]
recordsFilter := record.Filters()[1]
if resourceFilter.From() != eacl.HeaderFromService || recordsFilter.From() != eacl.HeaderFromService ||
resourceFilter.Matcher() != eacl.MatchUnknown || recordsFilter.Matcher() != eacl.MatchUnknown ||
resourceFilter.Key() != serviceRecordResourceKey || recordsFilter.Key() != serviceRecordGroupLengthKey {
return nil
}
groupLength, err := strconv.Atoi(recordsFilter.Value())
if err != nil {
return nil
}
return &ServiceRecord{
Resource: resourceFilter.Value(),
GroupRecordsLength: groupLength,
}
}
func formRecords(resource *astResource) ([]*eacl.Record, error) {
var res []*eacl.Record
for _, astOp := range resource.Operations {
for i := len(resource.Operations) - 1; i >= 0; i-- {
astOp := resource.Operations[i]
record := eacl.NewRecord()
record.SetOperation(astOp.Op)
record.SetAction(astOp.Action)
@ -888,19 +993,8 @@ func policyToAst(bktPolicy *bucketPolicy) (*ast, error) {
trimmedResource := strings.TrimPrefix(resource, arnAwsPrefix)
r, ok := rr[trimmedResource]
if !ok {
r = &astResource{resourceInfo: resourceInfo{Bucket: bktPolicy.Bucket}}
if trimmedResource != bktPolicy.Bucket {
versionedObject := strings.TrimPrefix(trimmedResource, bktPolicy.Bucket+"/")
objVersion := strings.Split(versionedObject, ":")
if len(objVersion) <= 2 {
r.Object = objVersion[0]
if len(objVersion) == 2 {
r.Version = objVersion[1]
}
} else {
r.Object = strings.Join(objVersion[:len(objVersion)-1], ":")
r.Version = objVersion[len(objVersion)-1]
}
r = &astResource{
resourceInfo: resourceInfoFromName(trimmedResource, bktPolicy.Bucket),
}
}
for _, action := range state.Action {
@ -921,6 +1015,25 @@ func policyToAst(bktPolicy *bucketPolicy) (*ast, error) {
return res, nil
}
func resourceInfoFromName(name, bucketName string) resourceInfo {
resInfo := resourceInfo{Bucket: bucketName}
if name != bucketName {
versionedObject := strings.TrimPrefix(name, bucketName+"/")
objVersion := strings.Split(versionedObject, ":")
if len(objVersion) <= 2 {
resInfo.Object = objVersion[0]
if len(objVersion) == 2 {
resInfo.Version = objVersion[1]
}
} else {
resInfo.Object = strings.Join(objVersion[:len(objVersion)-1], ":")
resInfo.Version = objVersion[len(objVersion)-1]
}
}
return resInfo
}
func astToPolicy(ast *ast) *bucketPolicy {
bktPolicy := &bucketPolicy{}
@ -1167,7 +1280,7 @@ func isWriteOperation(op eacl.Operation) bool {
return op == eacl.OperationDelete || op == eacl.OperationPut
}
func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, objectName string) *AccessControlPolicy {
func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, bucketName, objectVersion string) *AccessControlPolicy {
res := &AccessControlPolicy{
Owner: Owner{
ID: bucketACL.Info.Owner.String(),
@ -1177,38 +1290,27 @@ func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, objectName string)
m := make(map[string][]eacl.Operation)
for _, record := range bucketACL.EACL.Records() {
if len(record.Targets()) != 1 {
h.log.Warn("some acl not fully mapped")
astList := tableToAst(bucketACL.EACL, bucketName)
for _, resource := range astList.Resources {
if resource.Version != objectVersion {
continue
}
if objectName != "" {
var found bool
for _, filter := range record.Filters() {
if filter.Matcher() == eacl.MatchStringEqual &&
filter.Key() == object.AttributeFileName && filter.Value() == objectName {
found = true
}
}
if !found {
for _, op := range resource.Operations {
if op.Action != eacl.ActionAllow {
continue
}
}
target := record.Targets()[0]
if target.Role() == eacl.RoleOthers {
if record.Action() == eacl.ActionAllow {
list := append(m[allUsersGroup], record.Operation())
if len(op.Users) == 0 {
list := append(m[allUsersGroup], op.Op)
m[allUsersGroup] = list
} else {
for _, user := range op.Users {
list := append(m[user], op.Op)
m[user] = list
}
}
continue
}
for _, key := range target.BinaryKeys() {
id := hex.EncodeToString(key)
list := append(m[id], record.Operation())
m[id] = list
}
}
@ -1254,8 +1356,8 @@ func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, objectName string)
return res
}
func (h *handler) encodeBucketACL(bucketACL *layer.BucketACL) *AccessControlPolicy {
return h.encodeObjectACL(bucketACL, "")
func (h *handler) encodeBucketACL(bucketName string, bucketACL *layer.BucketACL) *AccessControlPolicy {
return h.encodeObjectACL(bucketACL, bucketName, "")
}
func contains(list []eacl.Operation, op eacl.Operation) bool {

88
api/handler/acl_test.go
View file

@ -456,42 +456,45 @@ func TestOrder(t *testing.T) {
Operations: []*astOperation{
{
Users: users,
Op: eacl.OperationGet,
Op: eacl.OperationPut,
Action: eacl.ActionAllow,
},
{
Op: eacl.OperationGet,
Op: eacl.OperationPut,
Action: eacl.ActionDeny,
},
},
},
},
}
record1 := eacl.NewRecord()
record1.SetOperation(eacl.OperationGet)
record1.SetAction(eacl.ActionAllow)
record1.SetTargets(*targetUser)
record1.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName)
record2 := eacl.NewRecord()
record2.SetOperation(eacl.OperationGet)
record2.SetAction(eacl.ActionDeny)
record2.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName)
record2.SetTargets(*targetOther)
record3 := eacl.NewRecord()
record3.SetOperation(eacl.OperationGet)
record3.SetAction(eacl.ActionAllow)
record3.SetTargets(*targetUser)
record4 := eacl.NewRecord()
record4.SetOperation(eacl.OperationGet)
record4.SetAction(eacl.ActionDeny)
record4.SetTargets(*targetOther)
bucketServiceRec := &ServiceRecord{Resource: expectedAst.Resources[0].Name(), GroupRecordsLength: 2}
bucketUsersGetRec := eacl.NewRecord()
bucketUsersGetRec.SetOperation(eacl.OperationGet)
bucketUsersGetRec.SetAction(eacl.ActionAllow)
bucketUsersGetRec.SetTargets(*targetUser)
bucketOtherGetRec := eacl.NewRecord()
bucketOtherGetRec.SetOperation(eacl.OperationGet)
bucketOtherGetRec.SetAction(eacl.ActionDeny)
bucketOtherGetRec.SetTargets(*targetOther)
objectServiceRec := &ServiceRecord{Resource: expectedAst.Resources[1].Name(), GroupRecordsLength: 2}
objectUsersPutRec := eacl.NewRecord()
objectUsersPutRec.SetOperation(eacl.OperationPut)
objectUsersPutRec.SetAction(eacl.ActionAllow)
objectUsersPutRec.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName)
objectUsersPutRec.SetTargets(*targetUser)
objectOtherPutRec := eacl.NewRecord()
objectOtherPutRec.SetOperation(eacl.OperationPut)
objectOtherPutRec.SetAction(eacl.ActionDeny)
objectOtherPutRec.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName)
objectOtherPutRec.SetTargets(*targetOther)
expectedEacl := eacl.NewTable()
expectedEacl.AddRecord(record1)
expectedEacl.AddRecord(record2)
expectedEacl.AddRecord(record3)
expectedEacl.AddRecord(record4)
expectedEacl.AddRecord(objectServiceRec.ToEACLRecord())
expectedEacl.AddRecord(objectOtherPutRec)
expectedEacl.AddRecord(objectUsersPutRec)
expectedEacl.AddRecord(bucketServiceRec.ToEACLRecord())
expectedEacl.AddRecord(bucketOtherGetRec)
expectedEacl.AddRecord(bucketUsersGetRec)
t.Run("astToTable order and vice versa", func(t *testing.T) {
actualEacl, err := astToTable(expectedAst)
@ -533,7 +536,7 @@ func TestOrder(t *testing.T) {
mergedEacl, err := astToTable(mergedAst)
require.NoError(t, err)
require.Equal(t, *childRecord, mergedEacl.Records()[0])
require.Equal(t, *childRecord, mergedEacl.Records()[1])
})
}
@ -639,19 +642,24 @@ func TestAstToTable(t *testing.T) {
}
expectedTable := eacl.NewTable()
record := eacl.NewRecord()
record.SetAction(eacl.ActionDeny)
record.SetOperation(eacl.OperationGet)
eacl.AddFormedTarget(record, eacl.RoleOthers)
record.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, "objectName")
expectedTable.AddRecord(record)
record2 := eacl.NewRecord()
record2.SetAction(eacl.ActionAllow)
record2.SetOperation(eacl.OperationPut)
serviceRec1 := &ServiceRecord{Resource: ast.Resources[0].Name(), GroupRecordsLength: 1}
record1 := eacl.NewRecord()
record1.SetAction(eacl.ActionAllow)
record1.SetOperation(eacl.OperationPut)
// Unknown role is used, because it is ignored when keys are set
eacl.AddFormedTarget(record2, eacl.RoleUnknown, *(*ecdsa.PublicKey)(key.PublicKey()))
eacl.AddFormedTarget(record1, eacl.RoleUnknown, *(*ecdsa.PublicKey)(key.PublicKey()))
serviceRec2 := &ServiceRecord{Resource: ast.Resources[1].Name(), GroupRecordsLength: 1}
record2 := eacl.NewRecord()
record2.SetAction(eacl.ActionDeny)
record2.SetOperation(eacl.OperationGet)
eacl.AddFormedTarget(record2, eacl.RoleOthers)
record2.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, "objectName")
expectedTable.AddRecord(serviceRec2.ToEACLRecord())
expectedTable.AddRecord(record2)
expectedTable.AddRecord(serviceRec1.ToEACLRecord())
expectedTable.AddRecord(record1)
actualTable, err := astToTable(ast)
require.NoError(t, err)
@ -878,7 +886,11 @@ func allowedTableForObject(t *testing.T, key *keys.PrivateKey, resInfo *resource
}
expectedTable := eacl.NewTable()
for _, op := range readOps {
serviceRec := &ServiceRecord{Resource: resInfo.Name(), GroupRecordsLength: len(readOps)}
expectedTable.AddRecord(serviceRec.ToEACLRecord())
for i := len(readOps) - 1; i >= 0; i-- {
op := readOps[i]
record := getAllowRecord(op, key.PublicKey())
if isVersion {
record.AddObjectIDFilter(eacl.MatchStringEqual, objID)