[#590] Use service records to save resource info

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-07-15 17:35:11 +03:00 committed by Alex Vanin
parent b144e50f7f
commit 1e26cf1541
2 changed files with 230 additions and 116 deletions

View file

@ -10,6 +10,8 @@ import (
stderrors "errors" stderrors "errors"
"fmt" "fmt"
"net/http" "net/http"
"sort"
"strconv"
"strings" "strings"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/crypto/keys"
@ -92,6 +94,11 @@ type principal struct {
CanonicalUser string `json:"CanonicalUser,omitempty"` CanonicalUser string `json:"CanonicalUser,omitempty"`
} }
type orderedAstResource struct {
Index int
Resource *astResource
}
type ast struct { type ast struct {
Resources []*astResource Resources []*astResource
} }
@ -131,6 +138,23 @@ func (a astOperation) IsGroupGrantee() bool {
return len(a.Users) == 0 return len(a.Users) == 0
} }
const (
serviceRecordResourceKey = "Resource"
serviceRecordGroupLengthKey = "GroupLength"
)
type ServiceRecord struct {
Resource string
GroupRecordsLength int
}
func (s ServiceRecord) ToEACLRecord() *eacl.Record {
serviceRecord := eacl.NewRecord()
serviceRecord.AddFilter(eacl.HeaderFromService, eacl.MatchUnknown, serviceRecordResourceKey, s.Resource)
serviceRecord.AddFilter(eacl.HeaderFromService, eacl.MatchUnknown, serviceRecordGroupLengthKey, strconv.Itoa(s.GroupRecordsLength))
return serviceRecord
}
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) { func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context()) reqInfo := api.GetReqInfo(r.Context())
@ -146,7 +170,7 @@ func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
if err = api.EncodeToResponse(w, h.encodeBucketACL(bucketACL)); err != nil { if err = api.EncodeToResponse(w, h.encodeBucketACL(bktInfo.Name, bucketACL)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err) h.logAndSendError(w, "something went wrong", reqInfo, err)
return return
} }
@ -268,8 +292,20 @@ func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
if err = api.EncodeToResponse(w, h.encodeObjectACL(bucketACL, reqInfo.ObjectName)); err != nil { prm := &layer.HeadObjectParams{
h.logAndSendError(w, "something went wrong", reqInfo, err) BktInfo: bktInfo,
Object: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
objInfo, err := h.obj.GetObjectInfo(r.Context(), prm)
if err != nil {
h.logAndSendError(w, "could not object info", reqInfo, err)
return
}
if err = api.EncodeToResponse(w, h.encodeObjectACL(bucketACL, reqInfo.BucketName, objInfo.Version())); err != nil {
h.logAndSendError(w, "failed to encode response", reqInfo, err)
} }
} }
@ -566,51 +602,87 @@ func addPredefinedACP(acp *AccessControlPolicy, cannedACL string) (*AccessContro
} }
func tableToAst(table *eacl.Table, bktName string) *ast { func tableToAst(table *eacl.Table, bktName string) *ast {
result := &ast{} resourceMap := make(map[string]orderedAstResource)
metResources := make(map[string]int)
for i := len(table.Records()) - 1; i >= 0; i-- { var groupRecordsLeft int
resName := bktName var currentResource orderedAstResource
var objectName string for i, record := range table.Records() {
var version string if serviceRec := tryServiceRecord(record); serviceRec != nil {
record := table.Records()[i] resInfo := resourceInfoFromName(serviceRec.Resource, bktName)
for _, filter := range record.Filters() { groupRecordsLeft = serviceRec.GroupRecordsLength
if filter.Matcher() == eacl.MatchStringEqual {
if filter.Key() == object.AttributeFileName {
objectName = filter.Value()
resName += "/" + objectName
} else if filter.Key() == v2acl.FilterObjectID {
version = filter.Value()
resName += "/" + version
}
}
}
idx, ok := metResources[resName]
if !ok {
resource := &astResource{resourceInfo: resourceInfo{
Bucket: bktName,
Object: objectName,
Version: version,
}}
result.Resources = append(result.Resources, resource)
idx = len(result.Resources) - 1
metResources[resName] = idx
}
for _, target := range record.Targets() { currentResource = getResourceOrCreate(resourceMap, i, resInfo)
result.Resources[idx].Operations = addToList(result.Resources[idx].Operations, record, target) resourceMap[resInfo.Name()] = currentResource
} else if groupRecordsLeft != 0 {
groupRecordsLeft--
addOperationsAndUpdateMap(currentResource, record, resourceMap)
} else {
resInfo := resInfoFromFilters(bktName, record.Filters())
resource := getResourceOrCreate(resourceMap, i, resInfo)
addOperationsAndUpdateMap(resource, record, resourceMap)
} }
} }
for _, res := range result.Resources { return &ast{
for i, j := 0, len(res.Operations)-1; i < j; i, j = i+1, j-1 { Resources: formReverseOrderResources(resourceMap),
res.Operations[i], res.Operations[j] = res.Operations[j], res.Operations[i]
} }
}
func formReverseOrderResources(resourceMap map[string]orderedAstResource) []*astResource {
orderedResources := make([]orderedAstResource, 0, len(resourceMap))
for _, resource := range resourceMap {
orderedResources = append(orderedResources, resource)
}
sort.Slice(orderedResources, func(i, j int) bool {
return orderedResources[i].Index >= orderedResources[j].Index // reverse order
})
result := make([]*astResource, len(orderedResources))
for i, ordered := range orderedResources {
res := ordered.Resource
for j, k := 0, len(res.Operations)-1; j < k; j, k = j+1, k-1 {
res.Operations[j], res.Operations[k] = res.Operations[k], res.Operations[j]
}
result[i] = res
} }
return result return result
} }
func addOperationsAndUpdateMap(orderedRes orderedAstResource, record eacl.Record, resMap map[string]orderedAstResource) {
for _, target := range record.Targets() {
orderedRes.Resource.Operations = addToList(orderedRes.Resource.Operations, record, target)
}
resMap[orderedRes.Resource.Name()] = orderedRes
}
func getResourceOrCreate(resMap map[string]orderedAstResource, index int, resInfo resourceInfo) orderedAstResource {
resource, ok := resMap[resInfo.Name()]
if !ok {
resource = orderedAstResource{
Index: index,
Resource: &astResource{resourceInfo: resInfo},
}
}
return resource
}
func resInfoFromFilters(bucketName string, filters []eacl.Filter) resourceInfo {
resInfo := resourceInfo{Bucket: bucketName}
for _, filter := range filters {
if filter.Matcher() == eacl.MatchStringEqual {
if filter.Key() == object.AttributeFileName {
resInfo.Object = filter.Value()
} else if filter.Key() == v2acl.FilterObjectID {
resInfo.Version = filter.Value()
}
}
}
return resInfo
}
func mergeAst(parent, child *ast) (*ast, bool) { func mergeAst(parent, child *ast) (*ast, bool) {
updated := false updated := false
for _, resource := range child.Resources { for _, resource := range child.Resources {
@ -788,6 +860,13 @@ func astToTable(ast *ast) (*eacl.Table, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("form records: %w", err) return nil, fmt.Errorf("form records: %w", err)
} }
serviceRecord := ServiceRecord{
Resource: ast.Resources[i].Name(),
GroupRecordsLength: len(records),
}
table.AddRecord(serviceRecord.ToEACLRecord())
for _, rec := range records { for _, rec := range records {
table.AddRecord(rec) table.AddRecord(rec)
} }
@ -796,10 +875,36 @@ func astToTable(ast *ast) (*eacl.Table, error) {
return table, nil return table, nil
} }
func tryServiceRecord(record eacl.Record) *ServiceRecord {
if record.Action() != eacl.ActionUnknown || len(record.Targets()) != 0 ||
len(record.Filters()) != 2 {
return nil
}
resourceFilter := record.Filters()[0]
recordsFilter := record.Filters()[1]
if resourceFilter.From() != eacl.HeaderFromService || recordsFilter.From() != eacl.HeaderFromService ||
resourceFilter.Matcher() != eacl.MatchUnknown || recordsFilter.Matcher() != eacl.MatchUnknown ||
resourceFilter.Key() != serviceRecordResourceKey || recordsFilter.Key() != serviceRecordGroupLengthKey {
return nil
}
groupLength, err := strconv.Atoi(recordsFilter.Value())
if err != nil {
return nil
}
return &ServiceRecord{
Resource: resourceFilter.Value(),
GroupRecordsLength: groupLength,
}
}
func formRecords(resource *astResource) ([]*eacl.Record, error) { func formRecords(resource *astResource) ([]*eacl.Record, error) {
var res []*eacl.Record var res []*eacl.Record
for _, astOp := range resource.Operations { for i := len(resource.Operations) - 1; i >= 0; i-- {
astOp := resource.Operations[i]
record := eacl.NewRecord() record := eacl.NewRecord()
record.SetOperation(astOp.Op) record.SetOperation(astOp.Op)
record.SetAction(astOp.Action) record.SetAction(astOp.Action)
@ -888,19 +993,8 @@ func policyToAst(bktPolicy *bucketPolicy) (*ast, error) {
trimmedResource := strings.TrimPrefix(resource, arnAwsPrefix) trimmedResource := strings.TrimPrefix(resource, arnAwsPrefix)
r, ok := rr[trimmedResource] r, ok := rr[trimmedResource]
if !ok { if !ok {
r = &astResource{resourceInfo: resourceInfo{Bucket: bktPolicy.Bucket}} r = &astResource{
if trimmedResource != bktPolicy.Bucket { resourceInfo: resourceInfoFromName(trimmedResource, bktPolicy.Bucket),
versionedObject := strings.TrimPrefix(trimmedResource, bktPolicy.Bucket+"/")
objVersion := strings.Split(versionedObject, ":")
if len(objVersion) <= 2 {
r.Object = objVersion[0]
if len(objVersion) == 2 {
r.Version = objVersion[1]
}
} else {
r.Object = strings.Join(objVersion[:len(objVersion)-1], ":")
r.Version = objVersion[len(objVersion)-1]
}
} }
} }
for _, action := range state.Action { for _, action := range state.Action {
@ -921,6 +1015,25 @@ func policyToAst(bktPolicy *bucketPolicy) (*ast, error) {
return res, nil return res, nil
} }
func resourceInfoFromName(name, bucketName string) resourceInfo {
resInfo := resourceInfo{Bucket: bucketName}
if name != bucketName {
versionedObject := strings.TrimPrefix(name, bucketName+"/")
objVersion := strings.Split(versionedObject, ":")
if len(objVersion) <= 2 {
resInfo.Object = objVersion[0]
if len(objVersion) == 2 {
resInfo.Version = objVersion[1]
}
} else {
resInfo.Object = strings.Join(objVersion[:len(objVersion)-1], ":")
resInfo.Version = objVersion[len(objVersion)-1]
}
}
return resInfo
}
func astToPolicy(ast *ast) *bucketPolicy { func astToPolicy(ast *ast) *bucketPolicy {
bktPolicy := &bucketPolicy{} bktPolicy := &bucketPolicy{}
@ -1167,7 +1280,7 @@ func isWriteOperation(op eacl.Operation) bool {
return op == eacl.OperationDelete || op == eacl.OperationPut return op == eacl.OperationDelete || op == eacl.OperationPut
} }
func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, objectName string) *AccessControlPolicy { func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, bucketName, objectVersion string) *AccessControlPolicy {
res := &AccessControlPolicy{ res := &AccessControlPolicy{
Owner: Owner{ Owner: Owner{
ID: bucketACL.Info.Owner.String(), ID: bucketACL.Info.Owner.String(),
@ -1177,38 +1290,27 @@ func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, objectName string)
m := make(map[string][]eacl.Operation) m := make(map[string][]eacl.Operation)
for _, record := range bucketACL.EACL.Records() { astList := tableToAst(bucketACL.EACL, bucketName)
if len(record.Targets()) != 1 {
h.log.Warn("some acl not fully mapped") for _, resource := range astList.Resources {
if resource.Version != objectVersion {
continue continue
} }
if objectName != "" { for _, op := range resource.Operations {
var found bool if op.Action != eacl.ActionAllow {
for _, filter := range record.Filters() {
if filter.Matcher() == eacl.MatchStringEqual &&
filter.Key() == object.AttributeFileName && filter.Value() == objectName {
found = true
}
}
if !found {
continue continue
} }
}
target := record.Targets()[0] if len(op.Users) == 0 {
if target.Role() == eacl.RoleOthers { list := append(m[allUsersGroup], op.Op)
if record.Action() == eacl.ActionAllow {
list := append(m[allUsersGroup], record.Operation())
m[allUsersGroup] = list m[allUsersGroup] = list
} else {
for _, user := range op.Users {
list := append(m[user], op.Op)
m[user] = list
} }
continue
} }
for _, key := range target.BinaryKeys() {
id := hex.EncodeToString(key)
list := append(m[id], record.Operation())
m[id] = list
} }
} }
@ -1254,8 +1356,8 @@ func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, objectName string)
return res return res
} }
func (h *handler) encodeBucketACL(bucketACL *layer.BucketACL) *AccessControlPolicy { func (h *handler) encodeBucketACL(bucketName string, bucketACL *layer.BucketACL) *AccessControlPolicy {
return h.encodeObjectACL(bucketACL, "") return h.encodeObjectACL(bucketACL, bucketName, "")
} }
func contains(list []eacl.Operation, op eacl.Operation) bool { func contains(list []eacl.Operation, op eacl.Operation) bool {

View file

@ -456,42 +456,45 @@ func TestOrder(t *testing.T) {
Operations: []*astOperation{ Operations: []*astOperation{
{ {
Users: users, Users: users,
Op: eacl.OperationGet, Op: eacl.OperationPut,
Action: eacl.ActionAllow, Action: eacl.ActionAllow,
}, },
{ {
Op: eacl.OperationGet, Op: eacl.OperationPut,
Action: eacl.ActionDeny, Action: eacl.ActionDeny,
}, },
}, },
}, },
}, },
} }
bucketServiceRec := &ServiceRecord{Resource: expectedAst.Resources[0].Name(), GroupRecordsLength: 2}
record1 := eacl.NewRecord() bucketUsersGetRec := eacl.NewRecord()
record1.SetOperation(eacl.OperationGet) bucketUsersGetRec.SetOperation(eacl.OperationGet)
record1.SetAction(eacl.ActionAllow) bucketUsersGetRec.SetAction(eacl.ActionAllow)
record1.SetTargets(*targetUser) bucketUsersGetRec.SetTargets(*targetUser)
record1.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName) bucketOtherGetRec := eacl.NewRecord()
record2 := eacl.NewRecord() bucketOtherGetRec.SetOperation(eacl.OperationGet)
record2.SetOperation(eacl.OperationGet) bucketOtherGetRec.SetAction(eacl.ActionDeny)
record2.SetAction(eacl.ActionDeny) bucketOtherGetRec.SetTargets(*targetOther)
record2.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName) objectServiceRec := &ServiceRecord{Resource: expectedAst.Resources[1].Name(), GroupRecordsLength: 2}
record2.SetTargets(*targetOther) objectUsersPutRec := eacl.NewRecord()
record3 := eacl.NewRecord() objectUsersPutRec.SetOperation(eacl.OperationPut)
record3.SetOperation(eacl.OperationGet) objectUsersPutRec.SetAction(eacl.ActionAllow)
record3.SetAction(eacl.ActionAllow) objectUsersPutRec.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName)
record3.SetTargets(*targetUser) objectUsersPutRec.SetTargets(*targetUser)
record4 := eacl.NewRecord() objectOtherPutRec := eacl.NewRecord()
record4.SetOperation(eacl.OperationGet) objectOtherPutRec.SetOperation(eacl.OperationPut)
record4.SetAction(eacl.ActionDeny) objectOtherPutRec.SetAction(eacl.ActionDeny)
record4.SetTargets(*targetOther) objectOtherPutRec.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName)
objectOtherPutRec.SetTargets(*targetOther)
expectedEacl := eacl.NewTable() expectedEacl := eacl.NewTable()
expectedEacl.AddRecord(record1) expectedEacl.AddRecord(objectServiceRec.ToEACLRecord())
expectedEacl.AddRecord(record2) expectedEacl.AddRecord(objectOtherPutRec)
expectedEacl.AddRecord(record3) expectedEacl.AddRecord(objectUsersPutRec)
expectedEacl.AddRecord(record4) expectedEacl.AddRecord(bucketServiceRec.ToEACLRecord())
expectedEacl.AddRecord(bucketOtherGetRec)
expectedEacl.AddRecord(bucketUsersGetRec)
t.Run("astToTable order and vice versa", func(t *testing.T) { t.Run("astToTable order and vice versa", func(t *testing.T) {
actualEacl, err := astToTable(expectedAst) actualEacl, err := astToTable(expectedAst)
@ -533,7 +536,7 @@ func TestOrder(t *testing.T) {
mergedEacl, err := astToTable(mergedAst) mergedEacl, err := astToTable(mergedAst)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, *childRecord, mergedEacl.Records()[0]) require.Equal(t, *childRecord, mergedEacl.Records()[1])
}) })
} }
@ -639,19 +642,24 @@ func TestAstToTable(t *testing.T) {
} }
expectedTable := eacl.NewTable() expectedTable := eacl.NewTable()
record := eacl.NewRecord() serviceRec1 := &ServiceRecord{Resource: ast.Resources[0].Name(), GroupRecordsLength: 1}
record.SetAction(eacl.ActionDeny) record1 := eacl.NewRecord()
record.SetOperation(eacl.OperationGet) record1.SetAction(eacl.ActionAllow)
eacl.AddFormedTarget(record, eacl.RoleOthers) record1.SetOperation(eacl.OperationPut)
record.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, "objectName")
expectedTable.AddRecord(record)
record2 := eacl.NewRecord()
record2.SetAction(eacl.ActionAllow)
record2.SetOperation(eacl.OperationPut)
// Unknown role is used, because it is ignored when keys are set // Unknown role is used, because it is ignored when keys are set
eacl.AddFormedTarget(record2, eacl.RoleUnknown, *(*ecdsa.PublicKey)(key.PublicKey())) eacl.AddFormedTarget(record1, eacl.RoleUnknown, *(*ecdsa.PublicKey)(key.PublicKey()))
serviceRec2 := &ServiceRecord{Resource: ast.Resources[1].Name(), GroupRecordsLength: 1}
record2 := eacl.NewRecord()
record2.SetAction(eacl.ActionDeny)
record2.SetOperation(eacl.OperationGet)
eacl.AddFormedTarget(record2, eacl.RoleOthers)
record2.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, "objectName")
expectedTable.AddRecord(serviceRec2.ToEACLRecord())
expectedTable.AddRecord(record2) expectedTable.AddRecord(record2)
expectedTable.AddRecord(serviceRec1.ToEACLRecord())
expectedTable.AddRecord(record1)
actualTable, err := astToTable(ast) actualTable, err := astToTable(ast)
require.NoError(t, err) require.NoError(t, err)
@ -878,7 +886,11 @@ func allowedTableForObject(t *testing.T, key *keys.PrivateKey, resInfo *resource
} }
expectedTable := eacl.NewTable() expectedTable := eacl.NewTable()
for _, op := range readOps { serviceRec := &ServiceRecord{Resource: resInfo.Name(), GroupRecordsLength: len(readOps)}
expectedTable.AddRecord(serviceRec.ToEACLRecord())
for i := len(readOps) - 1; i >= 0; i-- {
op := readOps[i]
record := getAllowRecord(op, key.PublicKey()) record := getAllowRecord(op, key.PublicKey())
if isVersion { if isVersion {
record.AddObjectIDFilter(eacl.MatchStringEqual, objID) record.AddObjectIDFilter(eacl.MatchStringEqual, objID)