acid-ant/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#1120] cli: Edit object nodes output

Browse source 
Print detailed information only.
Allow to output to JSON.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2024-05-07 12:12:34 +03:00
parent a45b548a6f
commit 368218f0cc
1 changed files with 154 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

238
cmd/frostfs-cli/modules/object/nodes.go
View file

@ -1,14 +1,16 @@
package object package object
import ( import (
"bytes"
"cmp"
"context" "context"
"crypto/ecdsa" "crypto/ecdsa"
"encoding/hex" "encoding/hex"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"strconv" "slices"
"sync" "sync"
"text/tabwriter"
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
@ -29,7 +31,6 @@ import (
const ( const (
verifyPresenceAllFlag = "verify-presence-all" verifyPresenceAllFlag = "verify-presence-all"
explainFlag = "explain"
) )
var errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint") var errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint")
@ -46,16 +47,30 @@ type ecHeader struct {
parent oid.ID parent oid.ID
} }
type boolError struct {
value bool
err error
}
type objectPlacement struct { type objectPlacement struct {
requiredNodes []netmapSDK.NodeInfo requiredNodes []netmapSDK.NodeInfo
confirmedNodes []netmapSDK.NodeInfo confirmedNodes []netmapSDK.NodeInfo
} }
type objectNodesResult struct {
errors []error
placements map[oid.ID]objectPlacement
}
type ObjNodesDataObject struct {
ObjectID string `json:"object_id"`
RequiredNodes []string `json:"required_nodes,omitempty"`
ConfirmedNodes []string `json:"confirmed_nodes,omitempty"`
ECParentObjectID *string `json:"ec_parent_object_id,omitempty"`
ECIndex *uint32 `json:"ec_index,omitempty"`
}
type objNodesResultJSON struct {
ObjectID string `json:"object_id"`
DataObjects []ObjNodesDataObject `json:"data_objects,omitempty"`
Errors []string `json:"errors,omitempty"`
}
var objectNodesCmd = &cobra.Command{ var objectNodesCmd = &cobra.Command{
Use: "nodes", Use: "nodes",
Short: "List of nodes where the object is stored", Short: "List of nodes where the object is stored",
@ -77,8 +92,8 @@ func initObjectNodesCmd() {
flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage)
_ = objectGetCmd.MarkFlagRequired(commonflags.OIDFlag) _ = objectGetCmd.MarkFlagRequired(commonflags.OIDFlag)
flags.Bool(verifyPresenceAllFlag, false, "Verify the actual presence of the object on all netmap nodes") flags.Bool(verifyPresenceAllFlag, false, "Verify the actual presence of the object on all netmap nodes.")
flags.Bool(explainFlag, false, "Show detailed information about the object placement") flags.Bool(commonflags.JSON, false, "Print information about the object placement as json.")
} }
func objectNodes(cmd *cobra.Command, _ []string) { func objectNodes(cmd *cobra.Command, _ []string) {
@ -93,11 +108,11 @@ func objectNodes(cmd *cobra.Command, _ []string) {
placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli) placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli)
requiredNodes, objectsPlacement := getRequiredPlacement(cmd, objects, placementPolicy, netmap) result := getRequiredPlacement(cmd, objects, placementPolicy, netmap)
actualPlacement := getActualPlacement(cmd, netmap, requiredNodes, pk, objects, objectsPlacement) getActualPlacement(cmd, netmap, pk, objects, result)
printPlacement(cmd, netmap, requiredNodes, actualPlacement, objID, objects, objectsPlacement) printPlacement(cmd, objID, objects, result)
} }
func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) []phyObject { func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) []phyObject {
@ -256,16 +271,17 @@ func getNetMap(ctx context.Context, cli *client.Client) (*netmapSDK.NetMap, erro
return &nm, nil return &nm, nil
} }
func getRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) (map[uint64]netmapSDK.NodeInfo, map[oid.ID]objectPlacement) { func getRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) *objectNodesResult {
if policy.IsECPlacement(placementPolicy) { if policy.IsECPlacement(placementPolicy) {
return getECRequiredPlacement(cmd, objects, placementPolicy, netmap) return getECRequiredPlacement(cmd, objects, placementPolicy, netmap)
} }
return getReplicaRequiredPlacement(cmd, objects, placementPolicy, netmap) return getReplicaRequiredPlacement(cmd, objects, placementPolicy, netmap)
} }
func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) (map[uint64]netmapSDK.NodeInfo, map[oid.ID]objectPlacement) { func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) *objectNodesResult {
nodes := make(map[uint64]netmapSDK.NodeInfo) result := &objectNodesResult{
objectsNodes := make(map[oid.ID]objectPlacement) placements: make(map[oid.ID]objectPlacement),
}
placementBuilder := placement.NewNetworkMapBuilder(netmap) placementBuilder := placement.NewNetworkMapBuilder(netmap)
for _, object := range objects { for _, object := range objects {
placement, err := placementBuilder.BuildPlacement(object.containerID, &object.objectID, placementPolicy) placement, err := placementBuilder.BuildPlacement(object.containerID, &object.objectID, placementPolicy)
@ -277,30 +293,30 @@ func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placem
if !object.storedOnAllContainerNodes && nodeIdx == numOfReplicas { if !object.storedOnAllContainerNodes && nodeIdx == numOfReplicas {
break break
} }
nodes[n.Hash()] = n
op := objectsNodes[object.objectID] op := result.placements[object.objectID]
op.requiredNodes = append(op.requiredNodes, n) op.requiredNodes = append(op.requiredNodes, n)
objectsNodes[object.objectID] = op result.placements[object.objectID] = op
nodeIdx++ nodeIdx++
} }
} }
} }
return nodes, objectsNodes return result
} }
func getECRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) (map[uint64]netmapSDK.NodeInfo, map[oid.ID]objectPlacement) { func getECRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) *objectNodesResult {
nodes := make(map[uint64]netmapSDK.NodeInfo) result := &objectNodesResult{
objectsNodes := make(map[oid.ID]objectPlacement) placements: make(map[oid.ID]objectPlacement),
for _, object := range objects {
getECRequiredPlacementInternal(cmd, object, placementPolicy, netmap, nodes, objectsNodes)
} }
return nodes, objectsNodes for _, object := range objects {
getECRequiredPlacementInternal(cmd, object, placementPolicy, netmap, result)
}
return result
} }
func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap, nodes map[uint64]netmapSDK.NodeInfo, objectNodes map[oid.ID]objectPlacement) { func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap, result *objectNodesResult) {
placementObjectID := object.objectID placementObjectID := object.objectID
if object.ecHeader != nil { if object.ecHeader != nil {
placementObjectID = object.ecHeader.parent placementObjectID = object.ecHeader.parent
@ -312,11 +328,9 @@ func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placem
for _, vector := range placement { for _, vector := range placement {
if object.storedOnAllContainerNodes { if object.storedOnAllContainerNodes {
for _, node := range vector { for _, node := range vector {
nodes[node.Hash()] = node op := result.placements[object.objectID]
op := objectNodes[object.objectID]
op.requiredNodes = append(op.requiredNodes, node) op.requiredNodes = append(op.requiredNodes, node)
objectNodes[object.objectID] = op result.placements[object.objectID] = op
} }
continue continue
} }
@ -325,30 +339,18 @@ func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placem
chunkIdx := int(object.ecHeader.index) chunkIdx := int(object.ecHeader.index)
nodeIdx := chunkIdx % len(vector) nodeIdx := chunkIdx % len(vector)
node := vector[nodeIdx] node := vector[nodeIdx]
nodes[node.Hash()] = node
op := objectNodes[object.objectID] op := result.placements[object.objectID]
op.requiredNodes = append(op.requiredNodes, node) op.requiredNodes = append(op.requiredNodes, node)
objectNodes[object.objectID] = op result.placements[object.objectID] = op
} }
} }
} }
func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa.PrivateKey, objects []phyObject, result *objectNodesResult) {
pk *ecdsa.PrivateKey, objects []phyObject, objectNodes map[oid.ID]objectPlacement,
) map[uint64]boolError {
result := make(map[uint64]boolError)
resultMtx := &sync.Mutex{} resultMtx := &sync.Mutex{}
var candidates []netmapSDK.NodeInfo candidates := getNodesToCheckObjectExistance(cmd, netmap, result)
checkAllNodes, _ := cmd.Flags().GetBool(verifyPresenceAllFlag)
if checkAllNodes {
candidates = netmap.Nodes()
} else {
for _, n := range requiredPlacement {
candidates = append(candidates, n)
}
}
eg, egCtx := errgroup.WithContext(cmd.Context()) eg, egCtx := errgroup.WithContext(cmd.Context())
for _, cand := range candidates { for _, cand := range candidates {
@ -359,26 +361,24 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPl
if err != nil { if err != nil {
resultMtx.Lock() resultMtx.Lock()
defer resultMtx.Unlock() defer resultMtx.Unlock()
result[cand.Hash()] = boolError{err: err} result.errors = append(result.errors, fmt.Errorf("failed to connect to node %s: %w", hex.EncodeToString(cand.PublicKey()), err))
return nil return nil
} }
for _, object := range objects { for _, object := range objects {
object := object object := object
eg.Go(func() error { eg.Go(func() error {
var v boolError stored, err := isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk)
v.value, v.err = isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk)
resultMtx.Lock() resultMtx.Lock()
defer resultMtx.Unlock() defer resultMtx.Unlock()
if v.err == nil && v.value { if err == nil && stored {
op := objectNodes[object.objectID] op := result.placements[object.objectID]
op.confirmedNodes = append(op.confirmedNodes, cand) op.confirmedNodes = append(op.confirmedNodes, cand)
objectNodes[object.objectID] = op result.placements[object.objectID] = op
} }
if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { if err != nil {
return nil result.errors = append(result.errors, fmt.Errorf("failed to check object %s existence on node %s: %w", object.objectID.EncodeToString(), hex.EncodeToString(cand.PublicKey()), err))
} }
result[cand.Hash()] = v
return nil return nil
}) })
} }
@ -387,7 +387,24 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPl
} }
commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", eg.Wait()) commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", eg.Wait())
return result }
func getNodesToCheckObjectExistance(cmd *cobra.Command, netmap *netmapSDK.NetMap, result *objectNodesResult) []netmapSDK.NodeInfo {
checkAllNodes, _ := cmd.Flags().GetBool(verifyPresenceAllFlag)
if checkAllNodes {
return netmap.Nodes()
}
var nodes []netmapSDK.NodeInfo
visited := make(map[uint64]struct{})
for _, p := range result.placements {
for _, node := range p.requiredNodes {
if _, ok := visited[node.Hash()]; !ok {
nodes = append(nodes, node)
visited[node.Hash()] = struct{}{}
}
}
}
return nodes
} }
func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.NodeInfo, pk *ecdsa.PrivateKey) (*client.Client, error) { func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.NodeInfo, pk *ecdsa.PrivateKey) (*client.Client, error) {
@ -445,33 +462,44 @@ func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID,
return false, err return false, err
} }
func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, func printPlacement(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) {
actualPlacement map[uint64]boolError, objID oid.ID, objects []phyObject, objectNodes map[oid.ID]objectPlacement, normilizeObjectNodesResult(objects, result)
) { if json, _ := cmd.Flags().GetBool(commonflags.JSON); json {
w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug) printObjectNodesAsJSON(cmd, objID, objects, result)
_, err := fmt.Fprintln(w, "Node ID\tShould contain object\tActually contains object\t") } else {
commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", err) printObjectNodesAsText(cmd, objID, objects, result)
for _, n := range netmap.Nodes() { }
nodeID := hex.EncodeToString(n.PublicKey()) }
_, required := requiredPlacement[n.Hash()]
actual, actualExists := actualPlacement[n.Hash()] func normilizeObjectNodesResult(objects []phyObject, result *objectNodesResult) {
actualStr := "" slices.SortFunc(objects, func(lhs, rhs phyObject) int {
if actualExists { if lhs.ecHeader == nil && rhs.ecHeader == nil {
if actual.err != nil { return bytes.Compare(lhs.objectID[:], rhs.objectID[:])
actualStr = fmt.Sprintf("error: %v", actual.err)
} else {
actualStr = strconv.FormatBool(actual.value)
}
} }
_, err := fmt.Fprintf(w, "%s\t%s\t%s\t\n", nodeID, strconv.FormatBool(required), actualStr) if lhs.ecHeader == nil {
commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", err) return -1
} }
commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", w.Flush()) if rhs.ecHeader == nil {
return 1
if explain, _ := cmd.Flags().GetBool(explainFlag); !explain { }
return if lhs.ecHeader.parent == rhs.ecHeader.parent {
return cmp.Compare(lhs.ecHeader.index, rhs.ecHeader.index)
}
return bytes.Compare(lhs.ecHeader.parent[:], rhs.ecHeader.parent[:])
})
for _, obj := range objects {
op := result.placements[obj.objectID]
slices.SortFunc(op.confirmedNodes, func(lhs, rhs netmapSDK.NodeInfo) int {
return bytes.Compare(lhs.PublicKey(), rhs.PublicKey())
})
slices.SortFunc(op.requiredNodes, func(lhs, rhs netmapSDK.NodeInfo) int {
return bytes.Compare(lhs.PublicKey(), rhs.PublicKey())
})
result.placements[obj.objectID] = op
} }
}
func printObjectNodesAsText(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) {
fmt.Fprintf(cmd.OutOrStdout(), "Object %s stores payload in %d data objects:\n", objID.EncodeToString(), len(objects)) fmt.Fprintf(cmd.OutOrStdout(), "Object %s stores payload in %d data objects:\n", objID.EncodeToString(), len(objects))
for _, object := range objects { for _, object := range objects {
@ -480,7 +508,7 @@ func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacem
fmt.Fprintf(cmd.OutOrStdout(), "\tEC index: %d\n", object.ecHeader.index) fmt.Fprintf(cmd.OutOrStdout(), "\tEC index: %d\n", object.ecHeader.index)
fmt.Fprintf(cmd.OutOrStdout(), "\tEC parent: %s\n", object.ecHeader.parent.EncodeToString()) fmt.Fprintf(cmd.OutOrStdout(), "\tEC parent: %s\n", object.ecHeader.parent.EncodeToString())
} }
op, ok := objectNodes[object.objectID] op, ok := result.placements[object.objectID]
if !ok { if !ok {
continue continue
} }
@ -491,10 +519,52 @@ func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacem
} }
} }
if len(op.confirmedNodes) > 0 { if len(op.confirmedNodes) > 0 {
fmt.Fprintf(cmd.OutOrStdout(), "\tActual nodes:\n") fmt.Fprintf(cmd.OutOrStdout(), "\tConfirmed nodes:\n")
for _, node := range op.confirmedNodes { for _, node := range op.confirmedNodes {
fmt.Fprintf(cmd.OutOrStdout(), "\t\t- %s\n", hex.EncodeToString(node.PublicKey())) fmt.Fprintf(cmd.OutOrStdout(), "\t\t- %s\n", hex.EncodeToString(node.PublicKey()))
} }
} }
} }
if len(result.errors) == 0 {
return
}
fmt.Fprintf(cmd.OutOrStdout(), "Errors:\n")
for _, err := range result.errors {
fmt.Fprintf(cmd.OutOrStdout(), "\t%s\n", err.Error())
}
}
func printObjectNodesAsJSON(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) {
jsonResult := &objNodesResultJSON{
ObjectID: objID.EncodeToString(),
}
for _, object := range objects {
do := ObjNodesDataObject{
ObjectID: object.objectID.EncodeToString(),
}
if object.ecHeader != nil {
do.ECIndex = &object.ecHeader.index
ecParent := object.ecHeader.parent.EncodeToString()
do.ECParentObjectID = &ecParent
}
op, ok := result.placements[object.objectID]
if !ok {
continue
}
for _, rn := range op.requiredNodes {
do.RequiredNodes = append(do.RequiredNodes, hex.EncodeToString(rn.PublicKey()))
}
for _, cn := range op.confirmedNodes {
do.ConfirmedNodes = append(do.ConfirmedNodes, hex.EncodeToString(cn.PublicKey()))
}
jsonResult.DataObjects = append(jsonResult.DataObjects, do)
}
for _, err := range result.errors {
jsonResult.Errors = append(jsonResult.Errors, err.Error())
}
b, err := json.Marshal(jsonResult)
commonCmd.ExitOnErr(cmd, "failed to marshal json: %w", err)
cmd.Println(string(b))
} }