forked from TrueCloudLab/frostfs-node
[#1164] cli: Improve object nodes
performance
Do complex EC object parts flatten concurrently. Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
6cf512e574
commit
069c1559cc
1 changed files with 59 additions and 20 deletions
|
@ -33,7 +33,10 @@ const (
|
||||||
verifyPresenceAllFlag = "verify-presence-all"
|
verifyPresenceAllFlag = "verify-presence-all"
|
||||||
)
|
)
|
||||||
|
|
||||||
var errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint")
|
var (
|
||||||
|
errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint")
|
||||||
|
errMalformedComplexObject = errors.New("object consists of EC and non EC parts")
|
||||||
|
)
|
||||||
|
|
||||||
type phyObject struct {
|
type phyObject struct {
|
||||||
containerID cid.ID
|
containerID cid.ID
|
||||||
|
@ -180,19 +183,25 @@ func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli
|
||||||
|
|
||||||
func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, members []oid.ID, prmHead internalclient.HeadObjectPrm) []phyObject {
|
func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, members []oid.ID, prmHead internalclient.HeadObjectPrm) []phyObject {
|
||||||
result := make([]phyObject, 0, len(members))
|
result := make([]phyObject, 0, len(members))
|
||||||
|
var resultGuard sync.Mutex
|
||||||
|
|
||||||
|
if len(members) == 0 {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
prmHead.SetRawFlag(true) // to get an error instead of whole object
|
||||||
|
|
||||||
|
first := members[0]
|
||||||
var addrObj oid.Address
|
var addrObj oid.Address
|
||||||
addrObj.SetContainer(cnrID)
|
addrObj.SetContainer(cnrID)
|
||||||
prmHead.SetRawFlag(true) // to get an error instead of whole object
|
addrObj.SetObject(first)
|
||||||
for _, partObjID := range members {
|
|
||||||
addrObj.SetObject(partObjID)
|
|
||||||
prmHead.SetAddress(addrObj)
|
prmHead.SetAddress(addrObj)
|
||||||
|
|
||||||
_, err := internalclient.HeadObject(cmd.Context(), prmHead)
|
_, err := internalclient.HeadObject(cmd.Context(), prmHead)
|
||||||
var ecInfoError *objectSDK.ECInfoError
|
var ecInfoError *objectSDK.ECInfoError
|
||||||
if errors.As(err, &ecInfoError) {
|
if errors.As(err, &ecInfoError) {
|
||||||
chunks := getECObjectChunks(cmd, cnrID, partObjID, ecInfoError)
|
chunks := getECObjectChunks(cmd, cnrID, first, ecInfoError)
|
||||||
result = append(result, chunks...)
|
result = append(result, chunks...)
|
||||||
continue
|
|
||||||
} else if err == nil { // not EC object, so all members must be phy objects
|
} else if err == nil { // not EC object, so all members must be phy objects
|
||||||
for _, member := range members {
|
for _, member := range members {
|
||||||
result = append(result, phyObject{
|
result = append(result, phyObject{
|
||||||
|
@ -200,10 +209,40 @@ func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, member
|
||||||
objectID: member,
|
objectID: member,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
break
|
return result
|
||||||
|
} else {
|
||||||
|
commonCmd.ExitOnErr(cmd, "failed to flatten parts of complex object: %w", err)
|
||||||
}
|
}
|
||||||
commonCmd.ExitOnErr(cmd, "failed to read EC chunk of complex object: %w", err)
|
|
||||||
|
eg, egCtx := errgroup.WithContext(cmd.Context())
|
||||||
|
for idx := 1; idx < len(members); idx++ {
|
||||||
|
partObjID := members[idx]
|
||||||
|
|
||||||
|
eg.Go(func() error {
|
||||||
|
partHeadPrm := prmHead
|
||||||
|
var partAddr oid.Address
|
||||||
|
partAddr.SetContainer(cnrID)
|
||||||
|
partAddr.SetObject(partObjID)
|
||||||
|
partHeadPrm.SetAddress(partAddr)
|
||||||
|
|
||||||
|
_, err := internalclient.HeadObject(egCtx, partHeadPrm)
|
||||||
|
var ecInfoError *objectSDK.ECInfoError
|
||||||
|
if errors.As(err, &ecInfoError) {
|
||||||
|
chunks := getECObjectChunks(cmd, cnrID, partObjID, ecInfoError)
|
||||||
|
|
||||||
|
resultGuard.Lock()
|
||||||
|
defer resultGuard.Unlock()
|
||||||
|
result = append(result, chunks...)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
} else if err == nil {
|
||||||
|
return errMalformedComplexObject
|
||||||
}
|
}
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
commonCmd.ExitOnErr(cmd, "failed to flatten parts of complex object: %w", eg.Wait())
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue