TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 105 Pull requests 13 Projects Releases 2 Activity Actions

Use containersOf in neofs-adm #200

Merged
fyrchik merged 2 commits from fyrchik/frostfs-node:fix-containers-of into master 2023-04-04 07:26:24 +00:00
Conversation 0 Commits 2 Files changed 1 +89 -57
1 changed files with 89 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

146
cmd/frostfs-adm/internal/modules/morph/container.go
View file

@ -41,16 +41,28 @@ func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker, c Client
return ch, nil
}
func getContainersList(inv *invoker.Invoker, ch util.Uint160) ([][]byte, error) {
res, err := inv.Call(ch, "list", "")
func iterateContainerList(inv *invoker.Invoker, ch util.Uint160, f func([]byte) error) error {
sid, r, err := unwrap.SessionIterator(inv.Call(ch, "containersOf", ""))
if err != nil {
return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
return fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
}
itm, err := unwrap.Item(res, err)
if _, ok := itm.(stackitem.Null); !ok {
return unwrap.ArrayOfBytes(res, err)
// Nothing bad, except live session on the server, do not report to the user.
defer func() { _ = inv.TerminateSession(sid) }()
carpawell marked this conversation as resolved
carpawell commented 2023-04-03 15:19:23 +00:00
Review
Copy link

not sure exactly, but not that necessary as i remember according to my new neo-go API discoverings (4 months ago, could be wrong), actors do not do that as i remember (again, could be wrong)

not sure exactly, but not that necessary as i remember according to my new neo-go API discoverings (4 months ago, could be wrong), actors do not do that as i remember (again, could be wrong)
fyrchik commented 2023-04-04 07:22:37 +00:00
Review
Copy link

That is why we ignore error.

That is why we ignore error.
carpawell commented 2023-04-04 07:23:29 +00:00
Review
Copy link

I meant that it looks like neo-go team also thinks that we can drop defer at all. But not a problem.

I meant that it looks like `neo-go` team also thinks that we can drop `defer` at all. But not a problem.
fyrchik commented 2023-04-04 07:24:58 +00:00
Review
Copy link

It costs us (almost) nothing and neo-go internals can change any time.

It costs us (almost) nothing and neo-go internals can change any time.
items, err := inv.TraverseIterator(sid, &r, 0)
for err == nil && len(items) != 0 {
for j := range items {
b, err := items[j].TryBytes()
if err != nil {
return fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
}
if err := f(b); err != nil {
return err
}
}
items, err = inv.TraverseIterator(sid, &r, 0)
}
return nil, nil
return err
}
func dumpContainers(cmd *cobra.Command, _ []string) error {
@ -71,56 +83,81 @@ func dumpContainers(cmd *cobra.Command, _ []string) error {
return fmt.Errorf("unable to get contaract hash: %w", err)
}
cids, err := getContainersList(inv, ch)
if err != nil {
return fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
}
isOK, err := getCIDFilterFunc(cmd)
if err != nil {
return err
}
var containers []*Container
bw := io.NewBufBinWriter()
for _, id := range cids {
if !isOK(id) {
continue
}
bw.Reset()
emit.AppCall(bw.BinWriter, ch, "get", callflag.All, id)
emit.AppCall(bw.BinWriter, ch, "eACL", callflag.All, id)
res, err := inv.Run(bw.Bytes())
if err != nil {
return fmt.Errorf("can't get container info: %w", err)
}
if len(res.Stack) != 2 {
return fmt.Errorf("%w: expected 2 items on stack", errInvalidContainerResponse)
}
cnt := new(Container)
err = cnt.FromStackItem(res.Stack[0])
if err != nil {
return fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
}
ea := new(EACL)
err = ea.FromStackItem(res.Stack[1])
if err != nil {
return fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
}
if len(ea.Value) != 0 {
cnt.EACL = ea
}
containers = append(containers, cnt)
}
out, err := json.Marshal(containers)
f, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0o660)
if err != nil {
return err
}
return os.WriteFile(filename, out, 0o660)
defer f.Close()
_, err = f.Write([]byte{'['})
if err != nil {
return err
}
written := 0
enc := json.NewEncoder(f)
bw := io.NewBufBinWriter()
iterErr := iterateContainerList(inv, ch, func(id []byte) error {
if !isOK(id) {
return nil
}
cnt, err := dumpSingleContainer(bw, ch, inv, id)
if err != nil {
return err
}
// Writing directly to the file is ok, because json.Encoder does no internal buffering.
if written != 0 {
_, err = f.Write([]byte{','})
aarifullin commented 2023-04-03 21:59:37 +00:00
Review
Copy link

Just for my curiosity.
Are cnt-s big? It they are not, wouldn't be better to collect them in one slice?

var cnts []Container

and then invoke data, err := json.Marshal(&cnts) and just ioutil.WriteFile() for once?

If they are big, then yes - stream marshalling is not supported by encoding/json and the such way is needed

Just for my curiosity. Are `cnt`-s big? It they are not, wouldn't be better to collect them in one slice? ``` var cnts []Container ``` and then invoke `data, err := json.Marshal(&cnts)` and just `ioutil.WriteFile()` for once? If they are big, then yes - stream marshalling is not supported by `encoding/json` and the such way is needed
fyrchik commented 2023-04-04 06:43:18 +00:00
Review
Copy link

It was done like this before.
We have no limits on the number of containers: there can be a lot of them and eACL can be big too (64kb is the limit, already happened in practice).

It was done like this before. We have no limits on the number of containers: there can be a lot of them and `eACL` can be big too (64kb is the limit, already happened in practice).
👍 1
if err != nil {
return err
}
}
written++
return enc.Encode(cnt)
})
if iterErr != nil {
return iterErr
}
_, err = f.Write([]byte{']'})
return err
}
func dumpSingleContainer(bw *io.BufBinWriter, ch util.Uint160, inv *invoker.Invoker, id []byte) (*Container, error) {
bw.Reset()
emit.AppCall(bw.BinWriter, ch, "get", callflag.All, id)
carpawell marked this conversation as resolved
carpawell commented 2023-04-04 07:21:08 +00:00
Review
Copy link

can we use (tuned) Call here?

can we use (tuned) `Call` here?
carpawell commented 2023-04-04 07:22:30 +00:00
Review
Copy link

mean Invoker's method

mean `Invoker`'s method
fyrchik commented 2023-04-04 07:22:59 +00:00
Review
Copy link

We can, but this PR is only about containersOf refactoring.

We can, but this PR is only about `containersOf` refactoring.
emit.AppCall(bw.BinWriter, ch, "eACL", callflag.All, id)
res, err := inv.Run(bw.Bytes())
if err != nil {
return nil, fmt.Errorf("can't get container info: %w", err)
}
if len(res.Stack) != 2 {
return nil, fmt.Errorf("%w: expected 2 items on stack", errInvalidContainerResponse)
}
cnt := new(Container)
err = cnt.FromStackItem(res.Stack[0])
if err != nil {
return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
}
ea := new(EACL)
err = ea.FromStackItem(res.Stack[1])
if err != nil {
return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
}
if len(ea.Value) != 0 {
cnt.EACL = ea
}
return cnt, nil
}
func listContainers(cmd *cobra.Command, _ []string) error {
@ -136,20 +173,15 @@ func listContainers(cmd *cobra.Command, _ []string) error {
return fmt.Errorf("unable to get contaract hash: %w", err)
}
cids, err := getContainersList(inv, ch)
if err != nil {
return fmt.Errorf("%w: %v", errInvalidContainerResponse, err)
}
for _, id := range cids {
return iterateContainerList(inv, ch, func(id []byte) error {
var idCnr cid.ID
err = idCnr.Decode(id)
if err != nil {
return fmt.Errorf("unable to decode container id: %w", err)
}
cmd.Println(idCnr)
}
return nil
return nil
})
}
func restoreContainers(cmd *cobra.Command, _ []string) error {