From 1e26cf1541110d77879c74c1e870fdf8976bb5bb Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 15 Jul 2022 17:35:11 +0300 Subject: [PATCH] [#590] Use service records to save resource info Signed-off-by: Denis Kirillov --- api/handler/acl.go | 258 ++++++++++++++++++++++++++++------------ api/handler/acl_test.go | 88 ++++++++------ 2 files changed, 230 insertions(+), 116 deletions(-) diff --git a/api/handler/acl.go b/api/handler/acl.go index 6aa3dcfb..e05c6118 100644 --- a/api/handler/acl.go +++ b/api/handler/acl.go @@ -10,6 +10,8 @@ import ( stderrors "errors" "fmt" "net/http" + "sort" + "strconv" "strings" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -92,6 +94,11 @@ type principal struct { CanonicalUser string `json:"CanonicalUser,omitempty"` } +type orderedAstResource struct { + Index int + Resource *astResource +} + type ast struct { Resources []*astResource } @@ -131,6 +138,23 @@ func (a astOperation) IsGroupGrantee() bool { return len(a.Users) == 0 } +const ( + serviceRecordResourceKey = "Resource" + serviceRecordGroupLengthKey = "GroupLength" +) + +type ServiceRecord struct { + Resource string + GroupRecordsLength int +} + +func (s ServiceRecord) ToEACLRecord() *eacl.Record { + serviceRecord := eacl.NewRecord() + serviceRecord.AddFilter(eacl.HeaderFromService, eacl.MatchUnknown, serviceRecordResourceKey, s.Resource) + serviceRecord.AddFilter(eacl.HeaderFromService, eacl.MatchUnknown, serviceRecordGroupLengthKey, strconv.Itoa(s.GroupRecordsLength)) + return serviceRecord +} + func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) { reqInfo := api.GetReqInfo(r.Context()) @@ -146,7 +170,7 @@ func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) { return } - if err = api.EncodeToResponse(w, h.encodeBucketACL(bucketACL)); err != nil { + if err = api.EncodeToResponse(w, h.encodeBucketACL(bktInfo.Name, bucketACL)); err != nil { h.logAndSendError(w, "something went wrong", reqInfo, err) return } @@ -268,8 +292,20 @@ func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) { return } - if err = api.EncodeToResponse(w, h.encodeObjectACL(bucketACL, reqInfo.ObjectName)); err != nil { - h.logAndSendError(w, "something went wrong", reqInfo, err) + prm := &layer.HeadObjectParams{ + BktInfo: bktInfo, + Object: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + } + + objInfo, err := h.obj.GetObjectInfo(r.Context(), prm) + if err != nil { + h.logAndSendError(w, "could not object info", reqInfo, err) + return + } + + if err = api.EncodeToResponse(w, h.encodeObjectACL(bucketACL, reqInfo.BucketName, objInfo.Version())); err != nil { + h.logAndSendError(w, "failed to encode response", reqInfo, err) } } @@ -566,51 +602,87 @@ func addPredefinedACP(acp *AccessControlPolicy, cannedACL string) (*AccessContro } func tableToAst(table *eacl.Table, bktName string) *ast { - result := &ast{} - metResources := make(map[string]int) + resourceMap := make(map[string]orderedAstResource) - for i := len(table.Records()) - 1; i >= 0; i-- { - resName := bktName - var objectName string - var version string - record := table.Records()[i] - for _, filter := range record.Filters() { - if filter.Matcher() == eacl.MatchStringEqual { - if filter.Key() == object.AttributeFileName { - objectName = filter.Value() - resName += "/" + objectName - } else if filter.Key() == v2acl.FilterObjectID { - version = filter.Value() - resName += "/" + version - } - } - } - idx, ok := metResources[resName] - if !ok { - resource := &astResource{resourceInfo: resourceInfo{ - Bucket: bktName, - Object: objectName, - Version: version, - }} - result.Resources = append(result.Resources, resource) - idx = len(result.Resources) - 1 - metResources[resName] = idx - } + var groupRecordsLeft int + var currentResource orderedAstResource + for i, record := range table.Records() { + if serviceRec := tryServiceRecord(record); serviceRec != nil { + resInfo := resourceInfoFromName(serviceRec.Resource, bktName) + groupRecordsLeft = serviceRec.GroupRecordsLength - for _, target := range record.Targets() { - result.Resources[idx].Operations = addToList(result.Resources[idx].Operations, record, target) + currentResource = getResourceOrCreate(resourceMap, i, resInfo) + resourceMap[resInfo.Name()] = currentResource + } else if groupRecordsLeft != 0 { + groupRecordsLeft-- + addOperationsAndUpdateMap(currentResource, record, resourceMap) + } else { + resInfo := resInfoFromFilters(bktName, record.Filters()) + resource := getResourceOrCreate(resourceMap, i, resInfo) + addOperationsAndUpdateMap(resource, record, resourceMap) } } - for _, res := range result.Resources { - for i, j := 0, len(res.Operations)-1; i < j; i, j = i+1, j-1 { - res.Operations[i], res.Operations[j] = res.Operations[j], res.Operations[i] + return &ast{ + Resources: formReverseOrderResources(resourceMap), + } +} + +func formReverseOrderResources(resourceMap map[string]orderedAstResource) []*astResource { + orderedResources := make([]orderedAstResource, 0, len(resourceMap)) + for _, resource := range resourceMap { + orderedResources = append(orderedResources, resource) + } + sort.Slice(orderedResources, func(i, j int) bool { + return orderedResources[i].Index >= orderedResources[j].Index // reverse order + }) + + result := make([]*astResource, len(orderedResources)) + for i, ordered := range orderedResources { + res := ordered.Resource + for j, k := 0, len(res.Operations)-1; j < k; j, k = j+1, k-1 { + res.Operations[j], res.Operations[k] = res.Operations[k], res.Operations[j] } + + result[i] = res } return result } +func addOperationsAndUpdateMap(orderedRes orderedAstResource, record eacl.Record, resMap map[string]orderedAstResource) { + for _, target := range record.Targets() { + orderedRes.Resource.Operations = addToList(orderedRes.Resource.Operations, record, target) + } + resMap[orderedRes.Resource.Name()] = orderedRes +} + +func getResourceOrCreate(resMap map[string]orderedAstResource, index int, resInfo resourceInfo) orderedAstResource { + resource, ok := resMap[resInfo.Name()] + if !ok { + resource = orderedAstResource{ + Index: index, + Resource: &astResource{resourceInfo: resInfo}, + } + } + return resource +} + +func resInfoFromFilters(bucketName string, filters []eacl.Filter) resourceInfo { + resInfo := resourceInfo{Bucket: bucketName} + for _, filter := range filters { + if filter.Matcher() == eacl.MatchStringEqual { + if filter.Key() == object.AttributeFileName { + resInfo.Object = filter.Value() + } else if filter.Key() == v2acl.FilterObjectID { + resInfo.Version = filter.Value() + } + } + } + + return resInfo +} + func mergeAst(parent, child *ast) (*ast, bool) { updated := false for _, resource := range child.Resources { @@ -788,6 +860,13 @@ func astToTable(ast *ast) (*eacl.Table, error) { if err != nil { return nil, fmt.Errorf("form records: %w", err) } + + serviceRecord := ServiceRecord{ + Resource: ast.Resources[i].Name(), + GroupRecordsLength: len(records), + } + table.AddRecord(serviceRecord.ToEACLRecord()) + for _, rec := range records { table.AddRecord(rec) } @@ -796,10 +875,36 @@ func astToTable(ast *ast) (*eacl.Table, error) { return table, nil } +func tryServiceRecord(record eacl.Record) *ServiceRecord { + if record.Action() != eacl.ActionUnknown || len(record.Targets()) != 0 || + len(record.Filters()) != 2 { + return nil + } + + resourceFilter := record.Filters()[0] + recordsFilter := record.Filters()[1] + if resourceFilter.From() != eacl.HeaderFromService || recordsFilter.From() != eacl.HeaderFromService || + resourceFilter.Matcher() != eacl.MatchUnknown || recordsFilter.Matcher() != eacl.MatchUnknown || + resourceFilter.Key() != serviceRecordResourceKey || recordsFilter.Key() != serviceRecordGroupLengthKey { + return nil + } + + groupLength, err := strconv.Atoi(recordsFilter.Value()) + if err != nil { + return nil + } + + return &ServiceRecord{ + Resource: resourceFilter.Value(), + GroupRecordsLength: groupLength, + } +} + func formRecords(resource *astResource) ([]*eacl.Record, error) { var res []*eacl.Record - for _, astOp := range resource.Operations { + for i := len(resource.Operations) - 1; i >= 0; i-- { + astOp := resource.Operations[i] record := eacl.NewRecord() record.SetOperation(astOp.Op) record.SetAction(astOp.Action) @@ -888,19 +993,8 @@ func policyToAst(bktPolicy *bucketPolicy) (*ast, error) { trimmedResource := strings.TrimPrefix(resource, arnAwsPrefix) r, ok := rr[trimmedResource] if !ok { - r = &astResource{resourceInfo: resourceInfo{Bucket: bktPolicy.Bucket}} - if trimmedResource != bktPolicy.Bucket { - versionedObject := strings.TrimPrefix(trimmedResource, bktPolicy.Bucket+"/") - objVersion := strings.Split(versionedObject, ":") - if len(objVersion) <= 2 { - r.Object = objVersion[0] - if len(objVersion) == 2 { - r.Version = objVersion[1] - } - } else { - r.Object = strings.Join(objVersion[:len(objVersion)-1], ":") - r.Version = objVersion[len(objVersion)-1] - } + r = &astResource{ + resourceInfo: resourceInfoFromName(trimmedResource, bktPolicy.Bucket), } } for _, action := range state.Action { @@ -921,6 +1015,25 @@ func policyToAst(bktPolicy *bucketPolicy) (*ast, error) { return res, nil } +func resourceInfoFromName(name, bucketName string) resourceInfo { + resInfo := resourceInfo{Bucket: bucketName} + if name != bucketName { + versionedObject := strings.TrimPrefix(name, bucketName+"/") + objVersion := strings.Split(versionedObject, ":") + if len(objVersion) <= 2 { + resInfo.Object = objVersion[0] + if len(objVersion) == 2 { + resInfo.Version = objVersion[1] + } + } else { + resInfo.Object = strings.Join(objVersion[:len(objVersion)-1], ":") + resInfo.Version = objVersion[len(objVersion)-1] + } + } + + return resInfo +} + func astToPolicy(ast *ast) *bucketPolicy { bktPolicy := &bucketPolicy{} @@ -1167,7 +1280,7 @@ func isWriteOperation(op eacl.Operation) bool { return op == eacl.OperationDelete || op == eacl.OperationPut } -func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, objectName string) *AccessControlPolicy { +func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, bucketName, objectVersion string) *AccessControlPolicy { res := &AccessControlPolicy{ Owner: Owner{ ID: bucketACL.Info.Owner.String(), @@ -1177,38 +1290,27 @@ func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, objectName string) m := make(map[string][]eacl.Operation) - for _, record := range bucketACL.EACL.Records() { - if len(record.Targets()) != 1 { - h.log.Warn("some acl not fully mapped") + astList := tableToAst(bucketACL.EACL, bucketName) + + for _, resource := range astList.Resources { + if resource.Version != objectVersion { continue } - if objectName != "" { - var found bool - for _, filter := range record.Filters() { - if filter.Matcher() == eacl.MatchStringEqual && - filter.Key() == object.AttributeFileName && filter.Value() == objectName { - found = true - } - } - if !found { + for _, op := range resource.Operations { + if op.Action != eacl.ActionAllow { continue } - } - target := record.Targets()[0] - if target.Role() == eacl.RoleOthers { - if record.Action() == eacl.ActionAllow { - list := append(m[allUsersGroup], record.Operation()) + if len(op.Users) == 0 { + list := append(m[allUsersGroup], op.Op) m[allUsersGroup] = list + } else { + for _, user := range op.Users { + list := append(m[user], op.Op) + m[user] = list + } } - continue - } - - for _, key := range target.BinaryKeys() { - id := hex.EncodeToString(key) - list := append(m[id], record.Operation()) - m[id] = list } } @@ -1254,8 +1356,8 @@ func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, objectName string) return res } -func (h *handler) encodeBucketACL(bucketACL *layer.BucketACL) *AccessControlPolicy { - return h.encodeObjectACL(bucketACL, "") +func (h *handler) encodeBucketACL(bucketName string, bucketACL *layer.BucketACL) *AccessControlPolicy { + return h.encodeObjectACL(bucketACL, bucketName, "") } func contains(list []eacl.Operation, op eacl.Operation) bool { diff --git a/api/handler/acl_test.go b/api/handler/acl_test.go index 731a0b25..f9580c6e 100644 --- a/api/handler/acl_test.go +++ b/api/handler/acl_test.go @@ -456,42 +456,45 @@ func TestOrder(t *testing.T) { Operations: []*astOperation{ { Users: users, - Op: eacl.OperationGet, + Op: eacl.OperationPut, Action: eacl.ActionAllow, }, { - Op: eacl.OperationGet, + Op: eacl.OperationPut, Action: eacl.ActionDeny, }, }, }, }, } - - record1 := eacl.NewRecord() - record1.SetOperation(eacl.OperationGet) - record1.SetAction(eacl.ActionAllow) - record1.SetTargets(*targetUser) - record1.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName) - record2 := eacl.NewRecord() - record2.SetOperation(eacl.OperationGet) - record2.SetAction(eacl.ActionDeny) - record2.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName) - record2.SetTargets(*targetOther) - record3 := eacl.NewRecord() - record3.SetOperation(eacl.OperationGet) - record3.SetAction(eacl.ActionAllow) - record3.SetTargets(*targetUser) - record4 := eacl.NewRecord() - record4.SetOperation(eacl.OperationGet) - record4.SetAction(eacl.ActionDeny) - record4.SetTargets(*targetOther) + bucketServiceRec := &ServiceRecord{Resource: expectedAst.Resources[0].Name(), GroupRecordsLength: 2} + bucketUsersGetRec := eacl.NewRecord() + bucketUsersGetRec.SetOperation(eacl.OperationGet) + bucketUsersGetRec.SetAction(eacl.ActionAllow) + bucketUsersGetRec.SetTargets(*targetUser) + bucketOtherGetRec := eacl.NewRecord() + bucketOtherGetRec.SetOperation(eacl.OperationGet) + bucketOtherGetRec.SetAction(eacl.ActionDeny) + bucketOtherGetRec.SetTargets(*targetOther) + objectServiceRec := &ServiceRecord{Resource: expectedAst.Resources[1].Name(), GroupRecordsLength: 2} + objectUsersPutRec := eacl.NewRecord() + objectUsersPutRec.SetOperation(eacl.OperationPut) + objectUsersPutRec.SetAction(eacl.ActionAllow) + objectUsersPutRec.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName) + objectUsersPutRec.SetTargets(*targetUser) + objectOtherPutRec := eacl.NewRecord() + objectOtherPutRec.SetOperation(eacl.OperationPut) + objectOtherPutRec.SetAction(eacl.ActionDeny) + objectOtherPutRec.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, objectName) + objectOtherPutRec.SetTargets(*targetOther) expectedEacl := eacl.NewTable() - expectedEacl.AddRecord(record1) - expectedEacl.AddRecord(record2) - expectedEacl.AddRecord(record3) - expectedEacl.AddRecord(record4) + expectedEacl.AddRecord(objectServiceRec.ToEACLRecord()) + expectedEacl.AddRecord(objectOtherPutRec) + expectedEacl.AddRecord(objectUsersPutRec) + expectedEacl.AddRecord(bucketServiceRec.ToEACLRecord()) + expectedEacl.AddRecord(bucketOtherGetRec) + expectedEacl.AddRecord(bucketUsersGetRec) t.Run("astToTable order and vice versa", func(t *testing.T) { actualEacl, err := astToTable(expectedAst) @@ -533,7 +536,7 @@ func TestOrder(t *testing.T) { mergedEacl, err := astToTable(mergedAst) require.NoError(t, err) - require.Equal(t, *childRecord, mergedEacl.Records()[0]) + require.Equal(t, *childRecord, mergedEacl.Records()[1]) }) } @@ -639,19 +642,24 @@ func TestAstToTable(t *testing.T) { } expectedTable := eacl.NewTable() - record := eacl.NewRecord() - record.SetAction(eacl.ActionDeny) - record.SetOperation(eacl.OperationGet) - eacl.AddFormedTarget(record, eacl.RoleOthers) - record.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, "objectName") - expectedTable.AddRecord(record) - - record2 := eacl.NewRecord() - record2.SetAction(eacl.ActionAllow) - record2.SetOperation(eacl.OperationPut) + serviceRec1 := &ServiceRecord{Resource: ast.Resources[0].Name(), GroupRecordsLength: 1} + record1 := eacl.NewRecord() + record1.SetAction(eacl.ActionAllow) + record1.SetOperation(eacl.OperationPut) // Unknown role is used, because it is ignored when keys are set - eacl.AddFormedTarget(record2, eacl.RoleUnknown, *(*ecdsa.PublicKey)(key.PublicKey())) + eacl.AddFormedTarget(record1, eacl.RoleUnknown, *(*ecdsa.PublicKey)(key.PublicKey())) + + serviceRec2 := &ServiceRecord{Resource: ast.Resources[1].Name(), GroupRecordsLength: 1} + record2 := eacl.NewRecord() + record2.SetAction(eacl.ActionDeny) + record2.SetOperation(eacl.OperationGet) + eacl.AddFormedTarget(record2, eacl.RoleOthers) + record2.AddObjectAttributeFilter(eacl.MatchStringEqual, object.AttributeFileName, "objectName") + + expectedTable.AddRecord(serviceRec2.ToEACLRecord()) expectedTable.AddRecord(record2) + expectedTable.AddRecord(serviceRec1.ToEACLRecord()) + expectedTable.AddRecord(record1) actualTable, err := astToTable(ast) require.NoError(t, err) @@ -878,7 +886,11 @@ func allowedTableForObject(t *testing.T, key *keys.PrivateKey, resInfo *resource } expectedTable := eacl.NewTable() - for _, op := range readOps { + serviceRec := &ServiceRecord{Resource: resInfo.Name(), GroupRecordsLength: len(readOps)} + expectedTable.AddRecord(serviceRec.ToEACLRecord()) + + for i := len(readOps) - 1; i >= 0; i-- { + op := readOps[i] record := getAllowRecord(op, key.PublicKey()) if isVersion { record.AddObjectIDFilter(eacl.MatchStringEqual, objID)