355 lines
11 KiB
Go
355 lines
11 KiB
Go
package object
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ecdsa"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"sync"
|
|
"text/tabwriter"
|
|
|
|
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
|
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
|
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
"github.com/spf13/cobra"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
const (
|
|
verifyPresenceAllFlag = "verify-presence-all"
|
|
)
|
|
|
|
var errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint")
|
|
|
|
type objectNodesInfo struct {
|
|
containerID cid.ID
|
|
objectID oid.ID
|
|
relatedObjectIDs []oid.ID
|
|
isLockOrTombstone bool
|
|
}
|
|
|
|
type boolError struct {
|
|
value bool
|
|
err error
|
|
}
|
|
|
|
var objectNodesCmd = &cobra.Command{
|
|
Use: "nodes",
|
|
Short: "List of nodes where the object is stored",
|
|
Long: `List of nodes where the object should be stored and where it is actually stored.
|
|
Lock objects must exist on all nodes of the container.
|
|
For complex objects, a node is considered to store an object if the node stores at least one part of the complex object.
|
|
By default, the actual storage of the object is checked only on the nodes that should store the object. To check all nodes, use the flag --verify-presence-all.`,
|
|
Run: objectNodes,
|
|
}
|
|
|
|
func initObjectNodesCmd() {
|
|
commonflags.Init(objectNodesCmd)
|
|
|
|
flags := objectNodesCmd.Flags()
|
|
|
|
flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage)
|
|
_ = objectGetCmd.MarkFlagRequired(commonflags.CIDFlag)
|
|
|
|
flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage)
|
|
_ = objectGetCmd.MarkFlagRequired(commonflags.OIDFlag)
|
|
|
|
flags.Bool("verify-presence-all", false, "Verify the actual presence of the object on all netmap nodes")
|
|
}
|
|
|
|
func objectNodes(cmd *cobra.Command, _ []string) {
|
|
var cnrID cid.ID
|
|
var objID oid.ID
|
|
readObjectAddress(cmd, &cnrID, &objID)
|
|
|
|
pk := key.GetOrGenerate(cmd)
|
|
cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC)
|
|
|
|
objectInfo := getObjectInfo(cmd, cnrID, objID, cli, pk)
|
|
|
|
placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli)
|
|
|
|
requiredPlacement := getRequiredPlacement(cmd, objectInfo, placementPolicy, netmap)
|
|
|
|
actualPlacement := getActualPlacement(cmd, netmap, requiredPlacement, pk, objectInfo)
|
|
|
|
printPlacement(cmd, netmap, requiredPlacement, actualPlacement)
|
|
}
|
|
|
|
func getObjectInfo(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) *objectNodesInfo {
|
|
var addrObj oid.Address
|
|
addrObj.SetContainer(cnrID)
|
|
addrObj.SetObject(objID)
|
|
|
|
var prmHead internalclient.HeadObjectPrm
|
|
prmHead.SetClient(cli)
|
|
prmHead.SetAddress(addrObj)
|
|
prmHead.SetRawFlag(true)
|
|
|
|
Prepare(cmd, &prmHead)
|
|
readSession(cmd, &prmHead, pk, cnrID, objID)
|
|
|
|
res, err := internalclient.HeadObject(cmd.Context(), prmHead)
|
|
if err == nil {
|
|
return &objectNodesInfo{
|
|
containerID: cnrID,
|
|
objectID: objID,
|
|
isLockOrTombstone: res.Header().Type() == objectSDK.TypeLock || res.Header().Type() == objectSDK.TypeTombstone,
|
|
}
|
|
}
|
|
|
|
var errSplitInfo *objectSDK.SplitInfoError
|
|
|
|
if !errors.As(err, &errSplitInfo) {
|
|
commonCmd.ExitOnErr(cmd, "failed to get object info: %w", err)
|
|
return nil
|
|
}
|
|
|
|
splitInfo := errSplitInfo.SplitInfo()
|
|
|
|
if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID); ok {
|
|
return &objectNodesInfo{
|
|
containerID: cnrID,
|
|
objectID: objID,
|
|
relatedObjectIDs: members,
|
|
}
|
|
}
|
|
|
|
if members, ok := tryGetSplitMembersBySplitID(cmd, splitInfo, cli, cnrID); ok {
|
|
return &objectNodesInfo{
|
|
containerID: cnrID,
|
|
objectID: objID,
|
|
relatedObjectIDs: members,
|
|
}
|
|
}
|
|
|
|
members := tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnrID, objID)
|
|
return &objectNodesInfo{
|
|
containerID: cnrID,
|
|
objectID: objID,
|
|
relatedObjectIDs: members,
|
|
}
|
|
}
|
|
|
|
func getPlacementPolicyAndNetmap(cmd *cobra.Command, cnrID cid.ID, cli *client.Client) (placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) {
|
|
eg, egCtx := errgroup.WithContext(cmd.Context())
|
|
eg.Go(func() (e error) {
|
|
placementPolicy, e = getPlacementPolicy(egCtx, cnrID, cli)
|
|
return
|
|
})
|
|
eg.Go(func() (e error) {
|
|
netmap, e = getNetMap(egCtx, cli)
|
|
return
|
|
})
|
|
commonCmd.ExitOnErr(cmd, "rpc error: %w", eg.Wait())
|
|
return
|
|
}
|
|
|
|
func getPlacementPolicy(ctx context.Context, cnrID cid.ID, cli *client.Client) (netmapSDK.PlacementPolicy, error) {
|
|
prm := internalclient.GetContainerPrm{
|
|
Client: cli,
|
|
ClientParams: client.PrmContainerGet{
|
|
ContainerID: &cnrID,
|
|
},
|
|
}
|
|
|
|
res, err := internalclient.GetContainer(ctx, prm)
|
|
if err != nil {
|
|
return netmapSDK.PlacementPolicy{}, err
|
|
}
|
|
|
|
return res.Container().PlacementPolicy(), nil
|
|
}
|
|
|
|
func getNetMap(ctx context.Context, cli *client.Client) (*netmapSDK.NetMap, error) {
|
|
var prm internalclient.NetMapSnapshotPrm
|
|
prm.SetClient(cli)
|
|
|
|
res, err := internalclient.NetMapSnapshot(ctx, prm)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
nm := res.NetMap()
|
|
return &nm, nil
|
|
}
|
|
|
|
func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo {
|
|
nodes := make(map[uint64]netmapSDK.NodeInfo)
|
|
placementBuilder := placement.NewNetworkMapBuilder(netmap)
|
|
placement, err := placementBuilder.BuildPlacement(objInfo.containerID, &objInfo.objectID, placementPolicy)
|
|
commonCmd.ExitOnErr(cmd, "failed to get required placement: %w", err)
|
|
for repIdx, rep := range placement {
|
|
numOfReplicas := placementPolicy.ReplicaNumberByIndex(repIdx)
|
|
var nodeIdx uint32
|
|
for _, n := range rep {
|
|
if !objInfo.isLockOrTombstone && nodeIdx == numOfReplicas { // lock and tombstone objects should be on all container nodes
|
|
break
|
|
}
|
|
nodes[n.Hash()] = n
|
|
nodeIdx++
|
|
}
|
|
}
|
|
|
|
for _, relatedObjID := range objInfo.relatedObjectIDs {
|
|
placement, err = placementBuilder.BuildPlacement(objInfo.containerID, &relatedObjID, placementPolicy)
|
|
commonCmd.ExitOnErr(cmd, "failed to get required placement for related object: %w", err)
|
|
for _, rep := range placement {
|
|
for _, n := range rep {
|
|
nodes[n.Hash()] = n
|
|
}
|
|
}
|
|
}
|
|
|
|
return nodes
|
|
}
|
|
|
|
func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo,
|
|
pk *ecdsa.PrivateKey, objInfo *objectNodesInfo,
|
|
) map[uint64]boolError {
|
|
result := make(map[uint64]boolError)
|
|
resultMtx := &sync.Mutex{}
|
|
|
|
var candidates []netmapSDK.NodeInfo
|
|
checkAllNodes, _ := cmd.Flags().GetBool(verifyPresenceAllFlag)
|
|
if checkAllNodes {
|
|
candidates = netmap.Nodes()
|
|
} else {
|
|
for _, n := range requiredPlacement {
|
|
candidates = append(candidates, n)
|
|
}
|
|
}
|
|
|
|
eg, egCtx := errgroup.WithContext(cmd.Context())
|
|
for _, cand := range candidates {
|
|
cand := cand
|
|
|
|
eg.Go(func() error {
|
|
cli, err := createClient(egCtx, cmd, cand, pk)
|
|
if err != nil {
|
|
resultMtx.Lock()
|
|
defer resultMtx.Unlock()
|
|
result[cand.Hash()] = boolError{err: err}
|
|
return nil
|
|
}
|
|
|
|
eg.Go(func() error {
|
|
var v boolError
|
|
v.value, v.err = isObjectStoredOnNode(egCtx, cmd, objInfo.containerID, objInfo.objectID, cli, pk)
|
|
resultMtx.Lock()
|
|
defer resultMtx.Unlock()
|
|
if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) {
|
|
return nil
|
|
}
|
|
result[cand.Hash()] = v
|
|
return nil
|
|
})
|
|
|
|
for _, rObjID := range objInfo.relatedObjectIDs {
|
|
rObjID := rObjID
|
|
eg.Go(func() error {
|
|
var v boolError
|
|
v.value, v.err = isObjectStoredOnNode(egCtx, cmd, objInfo.containerID, rObjID, cli, pk)
|
|
resultMtx.Lock()
|
|
defer resultMtx.Unlock()
|
|
if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) {
|
|
return nil
|
|
}
|
|
result[cand.Hash()] = v
|
|
return nil
|
|
})
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", eg.Wait())
|
|
return result
|
|
}
|
|
|
|
func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.NodeInfo, pk *ecdsa.PrivateKey) (*client.Client, error) {
|
|
var cli *client.Client
|
|
var addresses []string
|
|
candidate.IterateNetworkEndpoints(func(s string) bool {
|
|
addresses = append(addresses, s)
|
|
return false
|
|
})
|
|
addresses = append(addresses, candidate.ExternalAddresses()...)
|
|
var lastErr error
|
|
for _, address := range addresses {
|
|
var networkAddr network.Address
|
|
lastErr = networkAddr.FromString(address)
|
|
if lastErr != nil {
|
|
continue
|
|
}
|
|
cli, lastErr = internalclient.GetSDKClient(ctx, cmd, pk, networkAddr)
|
|
if lastErr == nil {
|
|
break
|
|
}
|
|
}
|
|
if lastErr != nil {
|
|
return nil, lastErr
|
|
}
|
|
if cli == nil {
|
|
return nil, errNoAvailableEndpoint
|
|
}
|
|
return cli, nil
|
|
}
|
|
|
|
func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) (bool, error) {
|
|
var addrObj oid.Address
|
|
addrObj.SetContainer(cnrID)
|
|
addrObj.SetObject(objID)
|
|
|
|
var prmHead internalclient.HeadObjectPrm
|
|
prmHead.SetClient(cli)
|
|
prmHead.SetAddress(addrObj)
|
|
|
|
Prepare(cmd, &prmHead)
|
|
prmHead.SetTTL(1)
|
|
readSession(cmd, &prmHead, pk, cnrID, objID)
|
|
|
|
res, err := internalclient.HeadObject(ctx, prmHead)
|
|
if err == nil && res != nil {
|
|
return true, nil
|
|
}
|
|
var notFound *apistatus.ObjectNotFound
|
|
var removed *apistatus.ObjectAlreadyRemoved
|
|
if errors.As(err, ¬Found) || errors.As(err, &removed) {
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
|
|
func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, actualPlacement map[uint64]boolError) {
|
|
w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug)
|
|
defer func() {
|
|
commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", w.Flush())
|
|
}()
|
|
fmt.Fprintln(w, "Node ID\tShould contain object\tActually contains object\t")
|
|
for _, n := range netmap.Nodes() {
|
|
nodeID := hex.EncodeToString(n.PublicKey())
|
|
_, required := requiredPlacement[n.Hash()]
|
|
actual, actualExists := actualPlacement[n.Hash()]
|
|
actualStr := ""
|
|
if actualExists {
|
|
if actual.err != nil {
|
|
actualStr = fmt.Sprintf("error: %v", actual.err)
|
|
} else {
|
|
actualStr = strconv.FormatBool(actual.value)
|
|
}
|
|
}
|
|
fmt.Fprintf(w, "%s\t%s\t%s\t\n", nodeID, strconv.FormatBool(required), actualStr)
|
|
}
|
|
}
|