forked from TrueCloudLab/frostfs-node
[#932] adm: Move dump/restore/list-containers to package container
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
This commit is contained in:
parent
be15eab82a
commit
36fd6c663c
3 changed files with 74 additions and 58 deletions
451
cmd/frostfs-adm/internal/modules/morph/container/container.go
Normal file
451
cmd/frostfs-adm/internal/modules/morph/container/container.go
Normal file
|
|
@ -0,0 +1,451 @@
|
|||
package container
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
|
||||
morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/hash"
|
||||
"github.com/nspcc-dev/neo-go/pkg/io"
|
||||
"github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker"
|
||||
"github.com/nspcc-dev/neo-go/pkg/rpcclient/management"
|
||||
"github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap"
|
||||
"github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag"
|
||||
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||
"github.com/nspcc-dev/neo-go/pkg/vm/emit"
|
||||
"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
var errInvalidContainerResponse = errors.New("invalid response from container contract")
|
||||
|
||||
func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker) (util.Uint160, error) {
|
||||
s, err := cmd.Flags().GetString(containerContractFlag)
|
||||
var ch util.Uint160
|
||||
if err == nil {
|
||||
ch, err = util.Uint160DecodeStringLE(s)
|
||||
}
|
||||
if err != nil {
|
||||
r := management.NewReader(inv)
|
||||
nnsCs, err := r.GetContractByID(1)
|
||||
if err != nil {
|
||||
return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err)
|
||||
}
|
||||
ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(morphUtil.ContainerContract))
|
||||
if err != nil {
|
||||
return util.Uint160{}, err
|
||||
}
|
||||
}
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func iterateContainerList(inv *invoker.Invoker, ch util.Uint160, f func([]byte) error) error {
|
||||
sid, r, err := unwrap.SessionIterator(inv.Call(ch, "containersOf", ""))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
|
||||
}
|
||||
// Nothing bad, except live session on the server, do not report to the user.
|
||||
defer func() { _ = inv.TerminateSession(sid) }()
|
||||
|
||||
items, err := inv.TraverseIterator(sid, &r, 0)
|
||||
for err == nil && len(items) != 0 {
|
||||
for j := range items {
|
||||
b, err := items[j].TryBytes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
|
||||
}
|
||||
if err := f(b); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
items, err = inv.TraverseIterator(sid, &r, 0)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func dumpContainers(cmd *cobra.Command, _ []string) error {
|
||||
filename, err := cmd.Flags().GetString(containerDumpFlag)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid filename: %w", err)
|
||||
}
|
||||
|
||||
c, err := morphUtil.GetN3Client(viper.GetViper())
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't create N3 client: %w", err)
|
||||
}
|
||||
|
||||
inv := invoker.New(c, nil)
|
||||
|
||||
ch, err := getContainerContractHash(cmd, inv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to get contaract hash: %w", err)
|
||||
}
|
||||
|
||||
isOK, err := getCIDFilterFunc(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0o660)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
_, err = f.Write([]byte{'['})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
written := 0
|
||||
enc := json.NewEncoder(f)
|
||||
bw := io.NewBufBinWriter()
|
||||
iterErr := iterateContainerList(inv, ch, func(id []byte) error {
|
||||
if !isOK(id) {
|
||||
return nil
|
||||
}
|
||||
|
||||
cnt, err := dumpSingleContainer(bw, ch, inv, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Writing directly to the file is ok, because json.Encoder does no internal buffering.
|
||||
if written != 0 {
|
||||
_, err = f.Write([]byte{','})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
written++
|
||||
return enc.Encode(cnt)
|
||||
})
|
||||
if iterErr != nil {
|
||||
return iterErr
|
||||
}
|
||||
|
||||
_, err = f.Write([]byte{']'})
|
||||
return err
|
||||
}
|
||||
|
||||
func dumpSingleContainer(bw *io.BufBinWriter, ch util.Uint160, inv *invoker.Invoker, id []byte) (*Container, error) {
|
||||
bw.Reset()
|
||||
emit.AppCall(bw.BinWriter, ch, "get", callflag.All, id)
|
||||
emit.AppCall(bw.BinWriter, ch, "eACL", callflag.All, id)
|
||||
res, err := inv.Run(bw.Bytes())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't get container info: %w", err)
|
||||
}
|
||||
if len(res.Stack) != 2 {
|
||||
return nil, fmt.Errorf("%w: expected 2 items on stack", errInvalidContainerResponse)
|
||||
}
|
||||
|
||||
cnt := new(Container)
|
||||
err = cnt.FromStackItem(res.Stack[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
|
||||
}
|
||||
|
||||
ea := new(EACL)
|
||||
err = ea.FromStackItem(res.Stack[1])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
|
||||
}
|
||||
if len(ea.Value) != 0 {
|
||||
cnt.EACL = ea
|
||||
}
|
||||
return cnt, nil
|
||||
}
|
||||
|
||||
func listContainers(cmd *cobra.Command, _ []string) error {
|
||||
c, err := morphUtil.GetN3Client(viper.GetViper())
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't create N3 client: %w", err)
|
||||
}
|
||||
|
||||
inv := invoker.New(c, nil)
|
||||
|
||||
ch, err := getContainerContractHash(cmd, inv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to get contaract hash: %w", err)
|
||||
}
|
||||
|
||||
return iterateContainerList(inv, ch, func(id []byte) error {
|
||||
var idCnr cid.ID
|
||||
err = idCnr.Decode(id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to decode container id: %w", err)
|
||||
}
|
||||
cmd.Println(idCnr)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func restoreContainers(cmd *cobra.Command, _ []string) error {
|
||||
filename, err := cmd.Flags().GetString(containerDumpFlag)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid filename: %w", err)
|
||||
}
|
||||
|
||||
wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer wCtx.Close()
|
||||
|
||||
containers, err := parseContainers(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ch, err := fetchContainerContractHash(wCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
isOK, err := getCIDFilterFunc(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = restoreOrPutContainers(containers, isOK, cmd, wCtx, ch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return wCtx.AwaitTx()
|
||||
}
|
||||
|
||||
func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd *cobra.Command, wCtx *morphUtil.InitializeContext, ch util.Uint160) error {
|
||||
bw := io.NewBufBinWriter()
|
||||
for _, cnt := range containers {
|
||||
hv := hash.Sha256(cnt.Value)
|
||||
if !isOK(hv[:]) {
|
||||
continue
|
||||
}
|
||||
bw.Reset()
|
||||
restored, err := isContainerRestored(cmd, wCtx, ch, bw, hv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if restored {
|
||||
continue
|
||||
}
|
||||
|
||||
bw.Reset()
|
||||
|
||||
putContainer(bw, ch, cnt)
|
||||
|
||||
if bw.Err != nil {
|
||||
panic(bw.Err)
|
||||
}
|
||||
|
||||
if err := wCtx.SendConsensusTx(bw.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func putContainer(bw *io.BufBinWriter, ch util.Uint160, cnt Container) {
|
||||
emit.AppCall(bw.BinWriter, ch, "put", callflag.All,
|
||||
cnt.Value, cnt.Signature, cnt.PublicKey, cnt.Token)
|
||||
if ea := cnt.EACL; ea != nil {
|
||||
emit.AppCall(bw.BinWriter, ch, "setEACL", callflag.All,
|
||||
ea.Value, ea.Signature, ea.PublicKey, ea.Token)
|
||||
}
|
||||
}
|
||||
|
||||
func isContainerRestored(cmd *cobra.Command, wCtx *morphUtil.InitializeContext, containerHash util.Uint160, bw *io.BufBinWriter, hashValue util.Uint256) (bool, error) {
|
||||
emit.AppCall(bw.BinWriter, containerHash, "get", callflag.All, hashValue.BytesBE())
|
||||
res, err := wCtx.Client.InvokeScript(bw.Bytes(), nil)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("can't check if container is already restored: %w", err)
|
||||
}
|
||||
if len(res.Stack) == 0 {
|
||||
return false, errors.New("empty stack")
|
||||
}
|
||||
|
||||
old := new(Container)
|
||||
if err := old.FromStackItem(res.Stack[0]); err != nil {
|
||||
return false, fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
|
||||
}
|
||||
if len(old.Value) != 0 {
|
||||
var id cid.ID
|
||||
id.SetSHA256(hashValue)
|
||||
cmd.Printf("Container %s is already deployed.\n", id)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func parseContainers(filename string) ([]Container, error) {
|
||||
data, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't read dump file: %w", err)
|
||||
}
|
||||
|
||||
var containers []Container
|
||||
err = json.Unmarshal(data, &containers)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't parse dump file: %w", err)
|
||||
}
|
||||
return containers, nil
|
||||
}
|
||||
|
||||
func fetchContainerContractHash(wCtx *morphUtil.InitializeContext) (util.Uint160, error) {
|
||||
r := management.NewReader(wCtx.ReadOnlyInvoker)
|
||||
nnsCs, err := r.GetContractByID(1)
|
||||
if err != nil {
|
||||
return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err)
|
||||
}
|
||||
|
||||
ch, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, morphUtil.DomainOf(morphUtil.ContainerContract))
|
||||
if err != nil {
|
||||
return util.Uint160{}, fmt.Errorf("can't fetch container contract hash: %w", err)
|
||||
}
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// Container represents container struct in contract storage.
|
||||
type Container struct {
|
||||
Value []byte `json:"value"`
|
||||
Signature []byte `json:"signature"`
|
||||
PublicKey []byte `json:"public_key"`
|
||||
Token []byte `json:"token"`
|
||||
EACL *EACL `json:"eacl"`
|
||||
}
|
||||
|
||||
// EACL represents extended ACL struct in contract storage.
|
||||
type EACL struct {
|
||||
Value []byte `json:"value"`
|
||||
Signature []byte `json:"signature"`
|
||||
PublicKey []byte `json:"public_key"`
|
||||
Token []byte `json:"token"`
|
||||
}
|
||||
|
||||
// ToStackItem implements stackitem.Convertible.
|
||||
func (c *Container) ToStackItem() (stackitem.Item, error) {
|
||||
return stackitem.NewStruct([]stackitem.Item{
|
||||
stackitem.NewByteArray(c.Value),
|
||||
stackitem.NewByteArray(c.Signature),
|
||||
stackitem.NewByteArray(c.PublicKey),
|
||||
stackitem.NewByteArray(c.Token),
|
||||
}), nil
|
||||
}
|
||||
|
||||
// FromStackItem implements stackitem.Convertible.
|
||||
func (c *Container) FromStackItem(item stackitem.Item) error {
|
||||
arr, ok := item.Value().([]stackitem.Item)
|
||||
if !ok || len(arr) != 4 {
|
||||
return errors.New("invalid stack item type")
|
||||
}
|
||||
|
||||
value, err := arr[0].TryBytes()
|
||||
if err != nil {
|
||||
return errors.New("invalid container value")
|
||||
}
|
||||
|
||||
sig, err := arr[1].TryBytes()
|
||||
if err != nil {
|
||||
return errors.New("invalid container signature")
|
||||
}
|
||||
|
||||
pub, err := arr[2].TryBytes()
|
||||
if err != nil {
|
||||
return errors.New("invalid container public key")
|
||||
}
|
||||
|
||||
tok, err := arr[3].TryBytes()
|
||||
if err != nil {
|
||||
return errors.New("invalid container token")
|
||||
}
|
||||
|
||||
c.Value = value
|
||||
c.Signature = sig
|
||||
c.PublicKey = pub
|
||||
c.Token = tok
|
||||
return nil
|
||||
}
|
||||
|
||||
// ToStackItem implements stackitem.Convertible.
|
||||
func (c *EACL) ToStackItem() (stackitem.Item, error) {
|
||||
return stackitem.NewStruct([]stackitem.Item{
|
||||
stackitem.NewByteArray(c.Value),
|
||||
stackitem.NewByteArray(c.Signature),
|
||||
stackitem.NewByteArray(c.PublicKey),
|
||||
stackitem.NewByteArray(c.Token),
|
||||
}), nil
|
||||
}
|
||||
|
||||
// FromStackItem implements stackitem.Convertible.
|
||||
func (c *EACL) FromStackItem(item stackitem.Item) error {
|
||||
arr, ok := item.Value().([]stackitem.Item)
|
||||
if !ok || len(arr) != 4 {
|
||||
return errors.New("invalid stack item type")
|
||||
}
|
||||
|
||||
value, err := arr[0].TryBytes()
|
||||
if err != nil {
|
||||
return errors.New("invalid eACL value")
|
||||
}
|
||||
|
||||
sig, err := arr[1].TryBytes()
|
||||
if err != nil {
|
||||
return errors.New("invalid eACL signature")
|
||||
}
|
||||
|
||||
pub, err := arr[2].TryBytes()
|
||||
if err != nil {
|
||||
return errors.New("invalid eACL public key")
|
||||
}
|
||||
|
||||
tok, err := arr[3].TryBytes()
|
||||
if err != nil {
|
||||
return errors.New("invalid eACL token")
|
||||
}
|
||||
|
||||
c.Value = value
|
||||
c.Signature = sig
|
||||
c.PublicKey = pub
|
||||
c.Token = tok
|
||||
return nil
|
||||
}
|
||||
|
||||
// getCIDFilterFunc returns filtering function for container IDs.
|
||||
// Raw byte slices are used because it works with structures returned
|
||||
// from contract.
|
||||
func getCIDFilterFunc(cmd *cobra.Command) (func([]byte) bool, error) {
|
||||
rawIDs, err := cmd.Flags().GetStringSlice(containerIDsFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rawIDs) == 0 {
|
||||
return func([]byte) bool { return true }, nil
|
||||
}
|
||||
|
||||
for i := range rawIDs {
|
||||
err := new(cid.ID).DecodeString(rawIDs[i])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't parse CID %s: %w", rawIDs[i], err)
|
||||
}
|
||||
}
|
||||
sort.Strings(rawIDs)
|
||||
return func(rawID []byte) bool {
|
||||
var v [32]byte
|
||||
copy(v[:], rawID)
|
||||
|
||||
var id cid.ID
|
||||
id.SetSHA256(v)
|
||||
idStr := id.EncodeToString()
|
||||
n := sort.Search(len(rawIDs), func(i int) bool { return rawIDs[i] >= idStr })
|
||||
return n < len(rawIDs) && rawIDs[n] == idStr
|
||||
}, nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue