[#1164] cli: Improve `object nodes` performance
DCO action / DCO (pull_request) Successful in 5m12s Details
Vulncheck / Vulncheck (pull_request) Successful in 5m35s Details
Build / Build Components (1.21) (pull_request) Successful in 6m3s Details
Build / Build Components (1.22) (pull_request) Successful in 5m59s Details
Tests and linters / gopls check (pull_request) Successful in 7m39s Details
Tests and linters / Staticcheck (pull_request) Successful in 8m28s Details
Tests and linters / Lint (pull_request) Successful in 8m48s Details
Pre-commit hooks / Pre-commit (pull_request) Successful in 10m50s Details
Tests and linters / Tests (1.21) (pull_request) Successful in 12m12s Details
Tests and linters / Tests with -race (pull_request) Successful in 12m13s Details
Tests and linters / Tests (1.22) (pull_request) Successful in 12m38s Details

Do complex EC object parts flatten concurrently.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
pull/1169/head
Dmitrii Stepanov 2024-06-07 16:18:31 +03:00
parent 6cf512e574
commit 069c1559cc
1 changed files with 59 additions and 20 deletions

View File

@ -33,7 +33,10 @@ const (
verifyPresenceAllFlag = "verify-presence-all"
)
var errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint")
var (
errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint")
errMalformedComplexObject = errors.New("object consists of EC and non EC parts")
)
type phyObject struct {
containerID cid.ID
@ -180,30 +183,66 @@ func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli
func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, members []oid.ID, prmHead internalclient.HeadObjectPrm) []phyObject {
result := make([]phyObject, 0, len(members))
var resultGuard sync.Mutex
if len(members) == 0 {
return result
}
prmHead.SetRawFlag(true) // to get an error instead of whole object
first := members[0]
var addrObj oid.Address
addrObj.SetContainer(cnrID)
prmHead.SetRawFlag(true) // to get an error instead of whole object
for _, partObjID := range members {
addrObj.SetObject(partObjID)
prmHead.SetAddress(addrObj)
addrObj.SetObject(first)
prmHead.SetAddress(addrObj)
_, err := internalclient.HeadObject(cmd.Context(), prmHead)
var ecInfoError *objectSDK.ECInfoError
if errors.As(err, &ecInfoError) {
chunks := getECObjectChunks(cmd, cnrID, partObjID, ecInfoError)
result = append(result, chunks...)
continue
} else if err == nil { // not EC object, so all members must be phy objects
for _, member := range members {
result = append(result, phyObject{
containerID: cnrID,
objectID: member,
})
}
break
_, err := internalclient.HeadObject(cmd.Context(), prmHead)
var ecInfoError *objectSDK.ECInfoError
if errors.As(err, &ecInfoError) {
chunks := getECObjectChunks(cmd, cnrID, first, ecInfoError)
result = append(result, chunks...)
} else if err == nil { // not EC object, so all members must be phy objects
for _, member := range members {
result = append(result, phyObject{
containerID: cnrID,
objectID: member,
})
}
commonCmd.ExitOnErr(cmd, "failed to read EC chunk of complex object: %w", err)
return result
} else {
commonCmd.ExitOnErr(cmd, "failed to flatten parts of complex object: %w", err)
}
eg, egCtx := errgroup.WithContext(cmd.Context())
for idx := 1; idx < len(members); idx++ {
partObjID := members[idx]
eg.Go(func() error {
partHeadPrm := prmHead
var partAddr oid.Address
partAddr.SetContainer(cnrID)
partAddr.SetObject(partObjID)
partHeadPrm.SetAddress(partAddr)
_, err := internalclient.HeadObject(egCtx, partHeadPrm)
var ecInfoError *objectSDK.ECInfoError
if errors.As(err, &ecInfoError) {
chunks := getECObjectChunks(cmd, cnrID, partObjID, ecInfoError)
resultGuard.Lock()
defer resultGuard.Unlock()
result = append(result, chunks...)
return nil
} else if err == nil {
return errMalformedComplexObject
}
return err
})
}
commonCmd.ExitOnErr(cmd, "failed to flatten parts of complex object: %w", eg.Wait())
return result
}