Compare commits

...

138 commits

Author SHA1 Message Date
148d68933b [#1573] node: Simplify bootstrapWithState()
After #1382 we have no need to use lambdas.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-20 08:17:05 +00:00
51ee132ea3
[#1342] network/cache: Add node address to error multiClient
Signed-off-by: Alexander Chuprov <a.chuprov@yadro.com>
2024-12-18 19:27:35 +03:00
226dd25dd0 [#1568] pilorama: Replace "containerID" with "container ID" in the error message
It is "container ID" in every other place.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-18 15:52:26 +00:00
bd0197eaa8 [#1568] storage: Remove "could not/can't/failed to" from error messages
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-18 15:52:26 +00:00
e44b84c18c
[#1569] cli: Remove unnecessary variable after refactoring
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-18 10:17:04 +03:00
bed49e6ace
[#1569] cli: Make --range flag required in object hash
Previously, `object head` was used if no range was provided.
This is wrong on multiple levels:
1. We print an error if the checksum is missing in header,
   even though taking hash is possible.
2. We silently ignore --salt parameter.
3. `--range` is required for Object.RANGEHASH RPC, custom logic for one
   specific usecase has no value.

So we make it required and make CLI command follow more closely
the FrostFS API.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-18 10:17:04 +03:00
df05057ed4 [#1452] container: Add ListStream method
* Added new method for listing containers to container service.
  It opens stream and sends containers in batches.

* Added TransportSplitter wrapper around ExecutionService to
  split container ID list read from contract in parts that are
  smaller than grpc max message size. Batch size can be changed
  in node configuration file (as in example config file).

* Changed `container list` implementaion in cli: now ListStream
  is called by default. Old List is called only if ListStream
  is not implemented.

* Changed `internalclient.ListContainersPrm`.`Account` to
  `OwnerID` since `client.PrmContainerList`.`Account` was
  renamed to `OwnerID` in sdk.

Signed-off-by: Ekaterina Lebedeva <ekaterina.lebedeva@yadro.com>
2024-12-17 16:22:43 +03:00
b6c8ebf493 [#1453] container: Replace sort.Slice with slices.SortFunc
* Replaced `sort.Slice` with `slices.SortFunc` in
  `ListContainersRes.SortedIDList()` as it is a bit faster,
  according to 15102e6dfd.

Signed-off-by: Ekaterina Lebedeva <ekaterina.lebedeva@yadro.com>
2024-12-17 13:33:43 +03:00
6e82661c35 [#1563] tree: Wrap only ChainRouterError erros with ObjectAccessDenied
* Such wrapping helps to differentiate logical check errors and server internal
  errors.

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-12-16 15:16:07 +03:00
1a091ea7bb [#1563] object: Wrap only ChainRouterError erros with ObjectAccessDenied
* Such wrapping helps to differentiate logical check errors and server internal
  errors.

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-12-16 15:15:25 +03:00
7ac3542714 [#1563] ape: Introduce ChainRouterError error type
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-12-16 15:12:30 +03:00
f0c43c8d80
[#1502] Use zap.Error for logging errors
Use `zap.Error` instead of `zap.String` for logging errors: change all expressions like
`zap.String("error", err.Error())` or `zap.String("err", err.Error())` to `zap.Error(err)`.
Leave similar expressions with other messages unchanged, for example,
`zap.String("last_error", lastErr.Error())` or `zap.String("reason", ctx.Err().Error())`.

This change was made by applying the following patch:
```diff
@@
var err expression
@@
-zap.String("error", err.Error())
+zap.Error(err)

@@
var err expression
@@
-zap.String("err", err.Error())
+zap.Error(err)
```

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-16 11:13:42 +03:00
8ba9f31fca
[#1510] metabase/test: Fix BenchmarkListWithCursor
- Fix misplaced `(*DB).Close` (broken after 47dcfa20f3)
- Use `errors.Is` for error checking (broken after fcdbf5e509)

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-13 13:19:15 +03:00
2af3409d39
[#1510] metabase/test: Fix BenchmarkGet
Fix misplaced `(*DB).Close` (broken after 47dcfa20f3)

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-13 13:18:43 +03:00
d165ac042c
[#1558] morph/client: Reuse notary rpcclient wrapper
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-12 15:30:12 +03:00
7151c71d51
[#1558] morph/client: Remove "could not"/"can't"/"failed to" from error messages
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-12 15:30:12 +03:00
91d9dc2676
[#1558] morph/event: Remove "could not" from error messages
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-12 15:30:12 +03:00
7853dbc315 [#1557] morph/event: Remove embedded structs from scriptHashWithValue
Also, make them public, because otherwise `unused` linter complains.
```
pkg/morph/event/utils.go:25:2  unused  field `typ` is unused
```
This complain is wrong, though: we _use_ `typ` field because the whole
struct is used as a map key.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-12 11:55:09 +00:00
3821645085
[#1555] engine: Refactor (*StorageEngine).GetLocks
Refactored after renaming the method to replace the confusing `locked`
variable with `locks`.

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-11 15:06:38 +03:00
72470d6b48
[#1555] local_object_storage: Rename method GetLocked -> GetLocks
Renamed to better reflect the method's purpose of returning locks
for the specified object.

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-11 15:06:37 +03:00
e9837bbcf9 [#1554] morph/event: Remove unused AlphabetUpdate event
Refs TrueCloudLab/frostfs-contract#138.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-11 12:01:17 +00:00
a641c91594 [#1550] Add CODEOWNERS
Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>
2024-12-11 10:34:57 +00:00
b1614a284d [#1546] morph/event: Export NotificationHandlerInfo fields
Hiding them achieves nothing, as the struct has no methods and is not
used concurrently.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-11 07:39:49 +00:00
d0ce835fbf [#1546] morph/event: Merge notification parser and handlers
They are decoupled, but it is an error to have a handler without a
corresponding parser. Register them together on the code level and get
rid of unreachable code.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-11 07:39:49 +00:00
dfa51048a8 [#1546] morph/event: Remove "is started" checks from event handler registrar
This codepath hides possible bugs in code.
All initialization function should run before init stage.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-11 07:39:49 +00:00
670305a721 [#1546] morph/event: Remove nil checks from event handler registrar
This codepath hides possible bugs in code.
We would rather panic then silently fail.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-11 07:39:49 +00:00
1f6cf57e30 [#1548] metabase: Check if EC parent is removed or expired
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-12-11 07:26:33 +00:00
386a12eea4 [#1548] engine: Rename parent -> ecParent
Parent could mean split parent or EC parent. In this case it is EC parent only.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-12-11 07:26:33 +00:00
15139d80c9 [#1548] policer: Do not replicate EC chunk if object already removed
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-12-11 07:26:33 +00:00
41da27dad5
[#1549] engine: Drop Async flag from evacuation parameters
Now it is only async evacuation.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-12-10 17:00:00 +03:00
ac0511d214
[#1549] controlSvc: Drop deprecated EvacuateShard rpc
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-12-10 16:59:52 +03:00
7e542906ef [#1539] go.mod: Bump frostfs-sdk-go version
* Also fix placement unit-test in object manager

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-12-06 15:29:37 +03:00
d1bc4351c3
[#1545] morph/event: Simplify frostfs contract event parsing
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-06 14:00:23 +03:00
1c12f23b84 [#1541] morph/event: Simplify netmap contract event parsing
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-06 10:26:39 +00:00
a353d45742 [#1541] morph/event: Simplify container contract event parsing
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-06 10:26:39 +00:00
d5c46d812a [#1541] go.mod: Update frostfs-contract
New version contains more idiomatic types in the auto-generated code.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-06 10:26:39 +00:00
d5d5ce2074 [#1541] morph/event: Simplify balance contract event parsing
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-06 10:26:39 +00:00
7df3520d48 [#1540] getSvc: Drop redundant returns
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-12-05 12:39:49 +00:00
5fe78e51d1 [#1540] getSvc: Do not log context canceled errors during EC assemble
Those errors are fired when it is enough chunks retrieved and error group
cancels other requests.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-12-05 12:39:49 +00:00
84b4051b4d
[#1538] morph/container: Make opts struct similar to that of other contracts
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-04 15:30:58 +03:00
6a51086030
[#1538] morph/client: Remove TryNotary() option from side-chain contracts
The notary is always enabled and this option does always work.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-04 15:30:58 +03:00
5c3b2d95ba
[#1538] node: Assume notary is enabled
Notaryless environments are not tested at all since a while.
We use neo-go only and it has notary contract enabled.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-04 15:30:58 +03:00
2d5d4093be
[#1537] morph: Use (user.ID).ScriptHash() where possible
Pick up changes from TrueCloudLab/frostfs-sdk-go#198.

gopatch:
```
@@
var user expression
@@
-address.StringToUint160(user.EncodeToString())
+user.ScriptHash()
```

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-04 13:25:44 +03:00
e3487d5af5 [#1535] morph: Unify test invoke error messages
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-04 09:50:20 +00:00
e37dcdf88b [#1535] morph/netmap: Unify error messages for config retrieval
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-04 09:50:20 +00:00
6c679d1535 [#1535] morph: Unify client creation error messages
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-04 09:50:20 +00:00
281d65435e
[#1450] engine: Group object by shard before Inhume
```
goos: linux
goarch: amd64
pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine
cpu: 12th Gen Intel(R) Core(TM) i5-1235U
                                 │   old.txt    │              new.txt                │
                                 │    sec/op    │   sec/op     vs base                │
InhumeMultipart/objects=1-12        11.42m ± 1%   10.71m ± 0%   -6.27% (p=0.000 n=10)
InhumeMultipart/objects=10-12       113.5m ± 0%   100.9m ± 3%  -11.08% (p=0.000 n=10)
InhumeMultipart/objects=100-12     1135.4m ± 1%   681.3m ± 2%  -40.00% (p=0.000 n=10)
InhumeMultipart/objects=1000-12     11.358 ± 0%    1.089 ± 1%  -90.41% (p=0.000 n=10)
InhumeMultipart/objects=10000-12   113.251 ± 0%    1.645 ± 1%  -98.55% (p=0.000 n=10)
geomean                              1.136        265.5m       -76.63%
```

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-04 10:09:00 +03:00
b348b20289
[#1450] engine: Add benchmark for Inhume operation
Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-04 10:08:34 +03:00
748edd1999
[#1450] engine: Return shard-level error if object is expired on inhume
Since we have errors defined on the shard-level, it looks strage that we
check an error againt the shard-level error `ErrLockObjectRemoval`, but
then return the metabase-level error. Let's return the same shard-level
error instead.

Since we have errors defined on the shard-level

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-04 10:06:57 +03:00
47dfd8840c [#1532] node: Allow to omit metabase.path if shard is disabled
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-12-04 03:30:19 +00:00
432042c534
[#1527] engine: Add tests for handling expired objects on inhume and lock
Currently, it's allowed to inhume or lock an expired object.
Consider the following scenario:

1) An user inhumes or locks an object
2) The object expires
3) GC hasn't yet deleted the object
4) The node loses the associated tombstone or lock
5) Another node replicates tombstone or lock to the first node

In this case, the second node succeeds, which is the desired behavior.

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-03 12:29:45 +03:00
9cabca9dfe
[#1527] engine/test: Move default metabase options to separate function
Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-02 16:13:37 +03:00
60feed3b5f
[#1527] engine/test: Allow to specify current epoch in epochState
Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-12-02 15:37:25 +03:00
635a292ae4 [#1528] cli: Keep order for required nodes in the result of object nodes
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
2024-12-02 13:58:24 +03:00
edfa3f4825 [#1528] node: Keep order for equal elements when sort priority metrics
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
2024-12-02 13:58:19 +03:00
e0ac3a583f [#1523] metabase: Remove (*DB).IterateCoveredByTombstones
Remove this method because it isn't used anywhere since 7799f8e4c.

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-11-29 10:49:24 +00:00
00c608c05e [#1524] tree: Make check APE error get wrapped to api status
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-29 10:48:16 +00:00
bba1892fa1 [#1524] ape: Make APE checker return error without status
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-29 10:48:16 +00:00
01acec708f
[#1525] pilorama: Use AppendUint* helpers from stdlib
gopatch:
```
@@
var slice, e expression
@@
+import "encoding/binary"

-append(slice, byte(e), byte(e >> 8))
+binary.LittleEndian.AppendUint16(slice, e)
```

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-28 09:40:20 +03:00
aac65001e5 [#1522] adm/frostfsid: Remove unreachable condition
SendConsensusTx() modifies SendTxs field, if it is not the case, there
is a bug in code.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
1170370753 [#1522] adm/helper: Rename createSingleAccounts() -> getSingleAccounts()
It doesn't create any accounts, purely finds them in the wallet.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
9e275d44c8 [#1522] adm/helper: Unexport DefaultClientContext()
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
2469e0c683 [#1522] adm/helper: Remove NewActor() helper
It is used once, it is used only internally and it is single-statement.
I see no justification in having it as a separate function.
It introduces confusion, because we also have NewLocalActor().

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
a6ef4ab524 [#1522] adm/helper: Rename GetN3Client() -> NewRemoteClient()
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
49959c4166 [#1522] adm/helper: Unexport GetFrostfsIDAdmin()
It is used in `helper` package only, besides unit-tests.
Move unit-tests to the same package, where they belong.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
61ee1b5610 [#1522] adm: Simplify LocalClient.SendRawTransaction()
The old code was there before Copy() method was introduced.
It was also supposed to check errors, however, they are already checked
server-side.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
b10c954377 [#1522] adm: Split NewLocalClient() into functions
No functional changes.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
1605391628 [#1522] adm/helper: Simplify Client interface
Just reuse `actor.RPCActor`. No functional changes.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
b1766e47c7 [#1522] adm/helper: Remove unused GetCommittee() method from the Client interface
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
caa4253249 [#1522] adm: Remove unnecessary variable declaration
It is better to have small scope.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-26 08:13:35 +00:00
7eac5fb18b
Release v0.44.0
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-25 14:41:36 +03:00
0e5524dac7 [#1515] adm: Print address in base58 format in morph ape get-admin
Signed-off-by: George Bartolomey <george@bh4.ru>
2024-11-25 10:38:05 +00:00
3ebd560f42 [#1519] cli: Make descriptive help for--rule option
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-25 07:21:05 +00:00
1ed7ab75fb [#1517] cli: Print the reason of ape manager error
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-22 15:24:32 +03:00
99f9e59de9 [#1514] adm: Remove --alphabet-wallets flag from readonly commands
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-21 14:37:34 +00:00
256f96e252 [#1514] adm/nns: Rename getRPCClient() to nnsWriter()
Make it more specific and similar to nnsReader().

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-21 14:37:34 +00:00
e5ea95c045 [#1514] adm/nns: Do not return hash from getRPCClient()
It was unused and we employ better abstractions now.
gopatch:
```
@@
var a, b expression
@@
-a, b, _ := getRPCClient(...)
+a, b := getRPCClient(...)
```

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-21 14:37:34 +00:00
9073e555db [#1514] adm/nns: Do not create actor for readonly commands
`nns get-records` and `nns tokens` command do not need to sign anything,
so remove useless actor and use invoker directly.
`NewLocalActor()` is only used in `ape` and `nns` packages. `ape`
package seem to use it correctly, only when alphabet wallets are
provided, so no changes there.
Also, remove --alphabet-wallets flag from commands that do not need it.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-21 14:37:34 +00:00
2771fdb8c7 [#1514] adm/nns: Use nns.GetAllRecords() wrapper
It was not possible previously, because GetAllRecords() was not declared
safe in frostfs-contract.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-21 14:37:34 +00:00
efa4ce00b8 [#1514] go.mod: Update frostfs-contract to v0.21.0-rc.3
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-21 14:37:34 +00:00
f12f04199e [#1516] traverser: Check for placement vector out of range
Placement vector may contain fewer nodes count than it required by policy
due to the outage of the one of the node.

Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
2024-11-21 14:18:55 +03:00
2e2c62147d
[#1513] adm: Move ProtoConfigPath from constants to commonflags package
Refs #932

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-21 09:39:06 +03:00
6ae8667fb4
[#1509] .forgejo: Run actions on push to master
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-20 11:42:12 +03:00
49a4e727fd [#1507] timer/test: Use const for constants
Make it easy to see what the test is about.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-20 08:36:25 +00:00
2e974f734c [#1507] timer/test: Improve test coverage
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-20 08:36:25 +00:00
3042490340 [#1507] timer: Remove unused OnDelta() method
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-20 08:36:25 +00:00
a339b52a60 [#1501] adm: Refactor APE-chains managing subcommands
* Use `cmd/internal/common/ape` parser commands within `ape`
  subcommands
* Use flag names from `cmd/internal/common/ape

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-20 07:58:32 +00:00
4ab4ed6f96 [#1501] cli: Refactor bearer subcommand
* Use `cmd/internal/common/ape` parser commands within `generate-ape-override`
  subcommand
* Use flag names from `cmd/internal/common/ape`

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-20 07:58:32 +00:00
3b1364e4cf [#1501] cli: Refactor ape-manager subcommands
* Refactor ape-manager subcommands
* Use `cmd/internal/common/ape` parser commands within ape-manager subcommands
* Use flag names from `cmd/internal/common/ape`

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-20 07:58:32 +00:00
daff77b273 [#1501] cli: Refactor local override managing subcommands
* Refactor local override managing subcommands
* Use `cmd/internal/common/ape` parser commands within local
  override subcommands
* Use flag names from `cmd/internal/common/ape`

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-20 07:58:32 +00:00
7a7ee71a4d [#1501] cmd: Introduce common APE-chain parser commands
* Introduce common parsing commands to use them in `frostfs-cli`
  and `frostfs-adm` APE-related subcommands
* Introduce common flags for these parsing commands

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-20 07:58:32 +00:00
ffe9906266 [#1501] cli: Move APE-chain parser methods to pkg/util
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-20 07:58:32 +00:00
ae31ef3602 [#1501] cli: Move PrintHumanReadableAPEChain to a common package
* Both `frostfs-cli` and `frostfs-adm` APE-related subcommands use
  `PrintHumanReadableAPEChain` to print a parsed APE-chain. So, it's
  more correct to have it in a common package over `frostfs-cli` and
  `frostfs-adm` folders.

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-20 07:58:32 +00:00
e2cb0640f1 [#1501] util: Move eACL-to-APE converter to pkg/util
* `ConvertEACLToAPE` is useful method which couldn't be imported
  out of frostfs-node so far as it has been in `internal`
* Since `ConvertEACLToAPE` and related structures and unit-tests
  are placed in `pkg/util`

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-20 07:58:32 +00:00
9f4ce600ac
[#1505] adm: Allow to manage additional keys in frostfsid
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-19 16:25:16 +03:00
d82f0d1926
[#1496] node/control: Await until SetNetmapStatus() persists
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-15 16:36:07 +03:00
acd5babd86
[#1496] morph: Merge InvokeRes and WaitParams
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-15 16:36:07 +03:00
b65874d1c3
[#1496] morph: Return InvokeRes from all invoke*() methods
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-15 16:36:07 +03:00
69c63006da
[#1496] morph: Move tx waiter to morph package
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-15 16:36:07 +03:00
d77a218f7c [#1493] metabase: Merge Inhume() and DropGraves() for tombstones
DropGraves() is only used to drop gravemarks after a tombstone
removal. Thus, it makes sense to do Inhume() and DropGraves() in one
transaction. It has less overhead and no unexpected problems in case
of sudden power failure.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-14 06:47:04 +00:00
44df67492f [#1493] metabase: Split inhumeTx() into 2 functions
No functional changes.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-14 06:47:04 +00:00
1e6f132b4e [#1493] metabase: Pass InhumePrm by value
Unify with the other code, no functional changes.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-14 06:47:04 +00:00
6dc0dc6691 [#1493] shard: Take mode mutex in HandleExpiredTombstones()
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-14 06:47:04 +00:00
f7cb6b4d87 [#1482] Makefile: Update golangci-lint
Signed-off-by: Alexander Chuprov <a.chuprov@yadro.com>
2024-11-13 15:01:41 +00:00
7fc6101bec
[#1491] engine/test: Rework engine test utils
- Remove `testNewShard` and `setInitializedShards` because they
violated the default engine workflow. The correct workflow is:
first use `New()`, followed by `Open()`, and then `Init()`. As a
result, adding new logic to `(*StorageEngine).Init` caused several
tests to fail with a panic when attempting to access uninitialized
resources. Now, all engines created with the test utils must be
initialized manually. The new helper method `prepare` can be used
for that purpose.
- Additionally, `setInitializedShards` hardcoded the shard worker
pool size, which prevented it from being configured in tests and
benchmarks. This has been fixed as well.
- Ensure engine initialization is done wherever it was missing.
- Refactor `setShardsNumOpts`, `setShardsNumAdditionalOpts`, and
`setShardsNum`. Make them all depend on `setShardsNumOpts`.

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-11-13 14:42:53 +03:00
7ef36749d0
[#1491] engine/test: Move BenchmarkExists to exists_test.go
Move `BenchmarkExists` from `engine_test.go` to `exists_test.go`
for better organization and clarity.

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-11-13 14:09:29 +03:00
c6066d6ee4
[#1491] engine/test: Use more suitable testing utils here and there
Use `setShardsNum` instead of `setInitializedShards` wherever possible.

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
2024-11-13 14:09:29 +03:00
612b34d570
[#1437] logger: Add caller skip to log original caller position
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-13 10:36:12 +03:00
7429553266
[#1437] node: Fix contextcheck linter
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-13 10:36:10 +03:00
6921a89061
[#1437] ir: Fix contextcheck linters
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-13 10:36:10 +03:00
16598553d9
[#1437] shard: Fix contextcheck linter
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-13 10:36:09 +03:00
c139892117
[#1437] ir: Fix contextcheck linter
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-13 10:36:09 +03:00
62b5181618
[#1437] blobovnicza: Fix contextcheck linter
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-13 10:36:08 +03:00
6db46257c0
[#1437] node: Use ctx for logging
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-13 10:36:07 +03:00
c16dae8b4d
[#1437] logger: Use context to log trace id
Signed-off-by: Dmitrii Stepanov
2024-11-13 10:36:07 +03:00
fd004add00 [#1492] metabase: Fix import formatting
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-13 07:30:25 +00:00
8ed7a676d5 [#1492] metabase: Ensure Unmarshal() is called on a cloned slice
The slice returned from bucket.Get() is only valid during the tx
lifetime. Cloning it is not necessary everywhere, but better safe than
sorry.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-13 07:30:25 +00:00
b451de94c8 [#1492] metabase: Fix typo in objData
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-13 07:30:25 +00:00
f1556e3c42
[#1488] Makefile: Drop all containers created on env-up
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-12 17:24:44 +03:00
e122ff6013
[#1488] dev: Add Jaeger image and enable tracing on debug
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-12 17:24:44 +03:00
e2658c7519
[#1488] tracing: KV attributes for spans from config
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-12 17:24:43 +03:00
c00f4bab18
[#1488] go.mod: Bump observability version
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-12 17:24:43 +03:00
46fef276b4 [#1449] tree: Log tree sync with Info level
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-12 12:11:07 +00:00
9bd05e94c8 [#1449] tree: Add ApplyBatch method
Concurrent Apply can lead to child node applies before parent, so
undo/redo operations will perform. This leads to performance degradation
in case of tree with many sublevels.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-12 12:11:07 +00:00
16830033f8 [#1483] cli: Remove --basic-acl flag
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-12 12:10:51 +00:00
1cf51a8079 [#1483] cli/docs: Remove set-eacl mention
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-12 12:10:51 +00:00
3324c26fd8 [#1483] morph: Remove container.GetEACL()
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-12 12:10:51 +00:00
a692298533 [#1483] node: Remove eACL cache
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-12 12:10:51 +00:00
be2753de00 [#1490] docs: Update description for object.get.priority
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
2024-11-12 13:35:13 +03:00
b543569c3f [#1486] node: Introduce dual service support
* Register GRPC services for both neo.fs.v2 and frost.fs namespaces
* Use this temporary solution until all nodes are updated

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-12 08:43:47 +00:00
80f8a8fd3a
[#1396] cli/playground: Refactor
Signed-off-by: Alexander Chuprov <a.chuprov@yadro.com>
2024-11-12 11:09:27 +03:00
2f3bc6eb84
[#1396] cli/playground: Improve terminal control key handling
Signed-off-by: Alexander Chuprov <a.chuprov@yadro.com>
2024-11-12 11:07:48 +03:00
8a57c78f5f
[#1484] engine: Fix engine metrics
1. Add forgotten metrics for client requests
2. Include execIfNotBlocked into metrics

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2024-11-11 12:59:20 +03:00
ad01fb958a [#1474] Stop using obsolete .github directory
This commit is a part of multi-repo cleanup effort:
TrueCloudLab/frostfs-infra#136

Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>
2024-11-11 06:44:13 +00:00
d336f2d487 [#1393] adm: Make NewLocalActor receive accout name
* Some RPC-clients for contracts require different wallet account types.
  Since, `Policy` contract gets `consensus` accounts while `NNS` gets
  `committee` accounts.

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2024-11-08 13:57:51 +00:00
c82c753e9f [#1480] objsvc: Remove useless stream wrappers
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-08 12:01:14 +00:00
f666898e5d [#1480] objsvc: Remove EACL checks
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-08 12:01:14 +00:00
b1a31281e4 [#1480] ape: Remove SoftAPECheck flag
Previous release was EACL-compatible.
Starting from now all EACL should've been migrated to APE chains.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2024-11-08 12:01:14 +00:00
504 changed files with 5420 additions and 6032 deletions

View file

Before

Width:  |  Height:  |  Size: 5.5 KiB

After

Width:  |  Height:  |  Size: 5.5 KiB

View file

@ -1,6 +1,10 @@
name: Build
on: [pull_request]
on:
pull_request:
push:
branches:
- master
jobs:
build:

View file

@ -1,5 +1,10 @@
name: Pre-commit hooks
on: [pull_request]
on:
pull_request:
push:
branches:
- master
jobs:
precommit:

View file

@ -1,5 +1,10 @@
name: Tests and linters
on: [pull_request]
on:
pull_request:
push:
branches:
- master
jobs:
lint:

View file

@ -1,5 +1,10 @@
name: Vulncheck
on: [pull_request]
on:
pull_request:
push:
branches:
- master
jobs:
vulncheck:

View file

@ -9,6 +9,30 @@ Changelog for FrostFS Node
### Removed
### Updated
## [v0.44.0] - 2024-25-11 - Rongbuk
### Added
- Allow to prioritize nodes during GET traversal via attributes (#1439)
- Add metrics for the frostfsid cache (#1464)
- Customize constant attributes attached to every tracing span (#1488)
- Manage additional keys in the `frostfsid` contract (#1505)
- Describe `--rule` flag in detail for `frostfs-cli ape-manager` subcommands (#1519)
### Changed
- Support richer interaction with the console in `frostfs-cli container policy-playground` (#1396)
- Print address in base58 format in `frostfs-adm morph policy set-admin` (#1515)
### Fixed
- Fix EC object search (#1408)
- Fix EC object put when one of the nodes is unavailable (#1427)
### Removed
- Drop most of the eACL-related code (#1425)
- Remove `--basic-acl` flag from `frostfs-cli container create` (#1483)
### Upgrading from v0.43.0
The metabase schema has changed completely, resync is required.
## [v0.42.0]
### Added

3
CODEOWNERS Normal file
View file

@ -0,0 +1,3 @@
.* @TrueCloudLab/storage-core-committers @TrueCloudLab/storage-core-developers
.forgejo/.* @potyarkin
Makefile @potyarkin

View file

@ -8,8 +8,8 @@ HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
GO_VERSION ?= 1.22
LINT_VERSION ?= 1.61.0
TRUECLOUDLAB_LINT_VERSION ?= 0.0.7
LINT_VERSION ?= 1.62.0
TRUECLOUDLAB_LINT_VERSION ?= 0.0.8
PROTOC_VERSION ?= 25.0
PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-sdk-go)
PROTOC_OS_VERSION=osx-x86_64
@ -282,7 +282,6 @@ env-up: all
# Shutdown dev environment
env-down:
docker compose -f dev/docker-compose.yml down
docker volume rm -f frostfs-node_neo-go
docker compose -f dev/docker-compose.yml down -v
rm -rf ./$(TMP_DIR)/state
rm -rf ./$(TMP_DIR)/storage

View file

@ -1,5 +1,5 @@
<p align="center">
<img src="./.github/logo.svg" width="500px" alt="FrostFS">
<img src="./.forgejo/logo.svg" width="500px" alt="FrostFS">
</p>
<p align="center">
@ -98,7 +98,7 @@ See `frostfs-contract`'s README.md for build instructions.
4. To create container and put object into it run (container and object IDs will be different):
```
./bin/frostfs-cli container create -r 127.0.0.1:8080 --wallet ./dev/wallet.json --policy "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" --basic-acl public-read-write --await
./bin/frostfs-cli container create -r 127.0.0.1:8080 --wallet ./dev/wallet.json --policy "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" --await
Enter password > <- press ENTER, the is no password for wallet
CID: CfPhEuHQ2PRvM4gfBQDC4dWZY3NccovyfcnEdiq2ixju

View file

@ -1 +1 @@
v0.42.0
v0.44.0

View file

@ -20,6 +20,7 @@ const (
AlphabetWalletsFlagDesc = "Path to alphabet wallets dir"
LocalDumpFlag = "local-dump"
ProtoConfigPath = "protocol"
ContractsInitFlag = "contracts"
ContractsInitFlagDesc = "Path to archive with compiled FrostFS contracts (the default is to fetch the latest release from the official repository)"
ContractsURLFlag = "contracts-url"

View file

@ -135,7 +135,7 @@ func createContainerInfoProvider(cli *client.Client) (container.InfoProvider, er
if err != nil {
return nil, fmt.Errorf("resolve container contract hash: %w", err)
}
cc, err := morphcontainer.NewFromMorph(cli, sh, 0, morphcontainer.TryNotary())
cc, err := morphcontainer.NewFromMorph(cli, sh, 0)
if err != nil {
return nil, fmt.Errorf("create morph container client: %w", err)
}

View file

@ -5,35 +5,19 @@ import (
"encoding/json"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags"
parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/encoding/address"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
const (
namespaceTarget = "namespace"
containerTarget = "container"
userTarget = "user"
groupTarget = "group"
jsonFlag = "json"
jsonFlagDesc = "Output rule chains in JSON format"
chainIDFlag = "chain-id"
chainIDDesc = "Rule chain ID"
ruleFlag = "rule"
ruleFlagDesc = "Rule chain in text format"
pathFlag = "path"
pathFlagDesc = "path to encoded chain in JSON or binary format"
targetNameFlag = "target-name"
targetNameDesc = "Resource name in APE resource name format"
targetTypeFlag = "target-type"
targetTypeDesc = "Resource type(container/namespace)"
addrAdminFlag = "addr"
addrAdminDesc = "The address of the admins wallet"
chainNameFlag = "chain-name"
chainNameFlagDesc = "Chain name(ingress|s3)"
jsonFlag = "json"
jsonFlagDesc = "Output rule chains in JSON format"
addrAdminFlag = "addr"
addrAdminDesc = "The address of the admins wallet"
)
var (
@ -101,17 +85,17 @@ func initAddRuleChainCmd() {
addRuleChainCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
addRuleChainCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc)
addRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc)
_ = addRuleChainCmd.MarkFlagRequired(targetTypeFlag)
addRuleChainCmd.Flags().String(targetNameFlag, "", targetNameDesc)
_ = addRuleChainCmd.MarkFlagRequired(targetNameFlag)
addRuleChainCmd.Flags().String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc)
_ = addRuleChainCmd.MarkFlagRequired(apeCmd.TargetTypeFlag)
addRuleChainCmd.Flags().String(apeCmd.TargetNameFlag, "", apeCmd.TargetTypeFlagDesc)
_ = addRuleChainCmd.MarkFlagRequired(apeCmd.TargetNameFlag)
addRuleChainCmd.Flags().String(chainIDFlag, "", chainIDDesc)
_ = addRuleChainCmd.MarkFlagRequired(chainIDFlag)
addRuleChainCmd.Flags().StringArray(ruleFlag, []string{}, ruleFlagDesc)
addRuleChainCmd.Flags().String(pathFlag, "", pathFlagDesc)
addRuleChainCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc)
addRuleChainCmd.MarkFlagsMutuallyExclusive(ruleFlag, pathFlag)
addRuleChainCmd.Flags().String(apeCmd.ChainIDFlag, "", apeCmd.ChainIDFlagDesc)
_ = addRuleChainCmd.MarkFlagRequired(apeCmd.ChainIDFlag)
addRuleChainCmd.Flags().StringArray(apeCmd.RuleFlag, []string{}, apeCmd.RuleFlagDesc)
addRuleChainCmd.Flags().String(apeCmd.PathFlag, "", apeCmd.PathFlagDesc)
addRuleChainCmd.Flags().String(apeCmd.ChainNameFlag, apeCmd.Ingress, apeCmd.ChainNameFlagDesc)
addRuleChainCmd.MarkFlagsMutuallyExclusive(apeCmd.RuleFlag, apeCmd.PathFlag)
}
func initRemoveRuleChainCmd() {
@ -120,26 +104,25 @@ func initRemoveRuleChainCmd() {
removeRuleChainCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
removeRuleChainCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc)
removeRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc)
_ = removeRuleChainCmd.MarkFlagRequired(targetTypeFlag)
removeRuleChainCmd.Flags().String(targetNameFlag, "", targetNameDesc)
_ = removeRuleChainCmd.MarkFlagRequired(targetNameFlag)
removeRuleChainCmd.Flags().String(chainIDFlag, "", chainIDDesc)
removeRuleChainCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc)
removeRuleChainCmd.Flags().String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc)
_ = removeRuleChainCmd.MarkFlagRequired(apeCmd.TargetTypeFlag)
removeRuleChainCmd.Flags().String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc)
_ = removeRuleChainCmd.MarkFlagRequired(apeCmd.TargetNameFlag)
removeRuleChainCmd.Flags().String(apeCmd.ChainIDFlag, "", apeCmd.ChainIDFlagDesc)
removeRuleChainCmd.Flags().String(apeCmd.ChainNameFlag, apeCmd.Ingress, apeCmd.ChainNameFlagDesc)
removeRuleChainCmd.Flags().Bool(commonflags.AllFlag, false, "Remove all chains for target")
removeRuleChainCmd.MarkFlagsMutuallyExclusive(commonflags.AllFlag, chainIDFlag)
removeRuleChainCmd.MarkFlagsMutuallyExclusive(commonflags.AllFlag, apeCmd.ChainIDFlag)
}
func initListRuleChainsCmd() {
Cmd.AddCommand(listRuleChainsCmd)
listRuleChainsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
listRuleChainsCmd.Flags().StringP(targetTypeFlag, "t", "", targetTypeDesc)
_ = listRuleChainsCmd.MarkFlagRequired(targetTypeFlag)
listRuleChainsCmd.Flags().String(targetNameFlag, "", targetNameDesc)
_ = listRuleChainsCmd.MarkFlagRequired(targetNameFlag)
listRuleChainsCmd.Flags().StringP(apeCmd.TargetTypeFlag, "t", "", apeCmd.TargetTypeFlagDesc)
_ = listRuleChainsCmd.MarkFlagRequired(apeCmd.TargetTypeFlag)
listRuleChainsCmd.Flags().String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc)
listRuleChainsCmd.Flags().Bool(jsonFlag, false, jsonFlagDesc)
listRuleChainsCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc)
listRuleChainsCmd.Flags().String(apeCmd.ChainNameFlag, apeCmd.Ingress, apeCmd.ChainNameFlagDesc)
}
func initSetAdminCmd() {
@ -161,15 +144,15 @@ func initListTargetsCmd() {
Cmd.AddCommand(listTargetsCmd)
listTargetsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
listTargetsCmd.Flags().StringP(targetTypeFlag, "t", "", targetTypeDesc)
_ = listTargetsCmd.MarkFlagRequired(targetTypeFlag)
listTargetsCmd.Flags().StringP(apeCmd.TargetTypeFlag, "t", "", apeCmd.TargetTypeFlagDesc)
_ = listTargetsCmd.MarkFlagRequired(apeCmd.TargetTypeFlag)
}
func addRuleChain(cmd *cobra.Command, _ []string) {
chain := parseChain(cmd)
chain := apeCmd.ParseChain(cmd)
target := parseTarget(cmd)
pci, ac := newPolicyContractInterface(cmd)
h, vub, err := pci.AddMorphRuleChain(parseChainName(cmd), target, chain)
h, vub, err := pci.AddMorphRuleChain(apeCmd.ParseChainName(cmd), target, chain)
cmd.Println("Waiting for transaction to persist...")
_, err = ac.Wait(h, vub, err)
commonCmd.ExitOnErr(cmd, "add rule chain error: %w", err)
@ -181,14 +164,14 @@ func removeRuleChain(cmd *cobra.Command, _ []string) {
pci, ac := newPolicyContractInterface(cmd)
removeAll, _ := cmd.Flags().GetBool(commonflags.AllFlag)
if removeAll {
h, vub, err := pci.RemoveMorphRuleChainsByTarget(parseChainName(cmd), target)
h, vub, err := pci.RemoveMorphRuleChainsByTarget(apeCmd.ParseChainName(cmd), target)
cmd.Println("Waiting for transaction to persist...")
_, err = ac.Wait(h, vub, err)
commonCmd.ExitOnErr(cmd, "remove rule chain error: %w", err)
cmd.Println("All chains for target removed successfully")
} else {
chainID := parseChainID(cmd)
h, vub, err := pci.RemoveMorphRuleChain(parseChainName(cmd), target, chainID)
chainID := apeCmd.ParseChainID(cmd)
h, vub, err := pci.RemoveMorphRuleChain(apeCmd.ParseChainName(cmd), target, chainID)
cmd.Println("Waiting for transaction to persist...")
_, err = ac.Wait(h, vub, err)
commonCmd.ExitOnErr(cmd, "remove rule chain error: %w", err)
@ -199,7 +182,7 @@ func removeRuleChain(cmd *cobra.Command, _ []string) {
func listRuleChains(cmd *cobra.Command, _ []string) {
target := parseTarget(cmd)
pci, _ := newPolicyContractReaderInterface(cmd)
chains, err := pci.ListMorphRuleChains(parseChainName(cmd), target)
chains, err := pci.ListMorphRuleChains(apeCmd.ParseChainName(cmd), target)
commonCmd.ExitOnErr(cmd, "list rule chains error: %w", err)
if len(chains) == 0 {
return
@ -210,14 +193,14 @@ func listRuleChains(cmd *cobra.Command, _ []string) {
prettyJSONFormat(cmd, chains)
} else {
for _, c := range chains {
parseutil.PrintHumanReadableAPEChain(cmd, c)
apeCmd.PrintHumanReadableAPEChain(cmd, c)
}
}
}
func setAdmin(cmd *cobra.Command, _ []string) {
s, _ := cmd.Flags().GetString(addrAdminFlag)
addr, err := util.Uint160DecodeStringLE(s)
addr, err := address.StringToUint160(s)
commonCmd.ExitOnErr(cmd, "can't decode admin addr: %w", err)
pci, ac := newPolicyContractInterface(cmd)
h, vub, err := pci.SetAdmin(addr)
@ -231,12 +214,11 @@ func getAdmin(cmd *cobra.Command, _ []string) {
pci, _ := newPolicyContractReaderInterface(cmd)
addr, err := pci.GetAdmin()
commonCmd.ExitOnErr(cmd, "unable to get admin: %w", err)
cmd.Println(addr.StringLE())
cmd.Println(address.Uint160ToString(addr))
}
func listTargets(cmd *cobra.Command, _ []string) {
typ, err := parseTargetType(cmd)
commonCmd.ExitOnErr(cmd, "parse target type error: %w", err)
typ := apeCmd.ParseTargetType(cmd)
pci, inv := newPolicyContractReaderInterface(cmd)
sid, it, err := pci.ListTargetsIterator(typ)

View file

@ -2,13 +2,12 @@ package ape
import (
"errors"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper"
parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
morph "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker"
@ -18,90 +17,29 @@ import (
"github.com/spf13/viper"
)
const (
ingress = "ingress"
s3 = "s3"
)
var mChainName = map[string]apechain.Name{
ingress: apechain.Ingress,
s3: apechain.S3,
}
var (
errUnknownTargetType = errors.New("unknown target type")
errChainIDCannotBeEmpty = errors.New("chain id cannot be empty")
errRuleIsNotParsed = errors.New("rule is not passed")
errUnsupportedChainName = errors.New("unsupported chain name")
)
var errUnknownTargetType = errors.New("unknown target type")
func parseTarget(cmd *cobra.Command) policyengine.Target {
name, _ := cmd.Flags().GetString(targetNameFlag)
typ, err := parseTargetType(cmd)
// interpret "root" namespace as empty
if typ == policyengine.Namespace && name == "root" {
name = ""
}
commonCmd.ExitOnErr(cmd, "read target type error: %w", err)
return policyengine.Target{
Name: name,
Type: typ,
}
}
func parseTargetType(cmd *cobra.Command) (policyengine.TargetType, error) {
typ, _ := cmd.Flags().GetString(targetTypeFlag)
typ := apeCmd.ParseTargetType(cmd)
name, _ := cmd.Flags().GetString(apeCmd.TargetNameFlag)
switch typ {
case namespaceTarget:
return policyengine.Namespace, nil
case containerTarget:
return policyengine.Container, nil
case userTarget:
return policyengine.User, nil
case groupTarget:
return policyengine.Group, nil
case policyengine.Namespace:
if name == "root" {
name = ""
}
return policyengine.NamespaceTarget(name)
case policyengine.Container:
var cnr cid.ID
commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(name))
return policyengine.ContainerTarget(name)
case policyengine.User:
return policyengine.UserTarget(name)
case policyengine.Group:
return policyengine.GroupTarget(name)
default:
commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType)
}
return -1, errUnknownTargetType
}
func parseChainID(cmd *cobra.Command) apechain.ID {
chainID, _ := cmd.Flags().GetString(chainIDFlag)
if chainID == "" {
commonCmd.ExitOnErr(cmd, "read chain id error: %w",
errChainIDCannotBeEmpty)
}
return apechain.ID(chainID)
}
func parseChain(cmd *cobra.Command) *apechain.Chain {
chain := new(apechain.Chain)
if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 {
commonCmd.ExitOnErr(cmd, "parser error: %w", parseutil.ParseAPEChain(chain, rules))
} else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" {
commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath))
} else {
commonCmd.ExitOnErr(cmd, "parser error: %w", errRuleIsNotParsed)
}
chain.ID = parseChainID(cmd)
cmd.Println("Parsed chain:")
parseutil.PrintHumanReadableAPEChain(cmd, chain)
return chain
}
func parseChainName(cmd *cobra.Command) apechain.Name {
chainName, _ := cmd.Flags().GetString(chainNameFlag)
apeChainName, ok := mChainName[strings.ToLower(chainName)]
if !ok {
commonCmd.ExitOnErr(cmd, "", errUnsupportedChainName)
}
return apeChainName
panic("unreachable")
}
// invokerAdapter adapats invoker.Invoker to ContractStorageInvoker interface.
@ -115,16 +53,15 @@ func (n *invokerAdapter) GetRPCInvoker() invoker.RPCInvoke {
}
func newPolicyContractReaderInterface(cmd *cobra.Command) (*morph.ContractStorageReader, *invoker.Invoker) {
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err)
inv := invoker.New(c, nil)
var ch util.Uint160
r := management.NewReader(inv)
nnsCs, err := helper.GetContractByID(r, 1)
commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err)
ch, err = helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.PolicyContract))
ch, err := helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.PolicyContract))
commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err)
invokerAdapter := &invokerAdapter{
@ -136,10 +73,10 @@ func newPolicyContractReaderInterface(cmd *cobra.Command) (*morph.ContractStorag
}
func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *helper.LocalActor) {
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err)
ac, err := helper.NewLocalActor(cmd, c)
ac, err := helper.NewLocalActor(cmd, c, constants.ConsensusAccountName)
commonCmd.ExitOnErr(cmd, "can't create actor: %w", err)
var ch util.Uint160

View file

@ -51,7 +51,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error {
nmHash util.Uint160
)
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
if err != nil {
return err
}

View file

@ -26,7 +26,7 @@ import (
const forceConfigSet = "force"
func dumpNetworkConfig(cmd *cobra.Command, _ []string) error {
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
if err != nil {
return fmt.Errorf("can't create N3 client: %w", err)
}

View file

@ -4,7 +4,6 @@ import "time"
const (
ConsensusAccountName = "consensus"
ProtoConfigPath = "protocol"
// MaxAlphabetNodes is the maximum number of candidates allowed, which is currently limited by the size
// of the invocation script.

View file

@ -76,7 +76,7 @@ func dumpContainers(cmd *cobra.Command, _ []string) error {
return fmt.Errorf("invalid filename: %w", err)
}
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
if err != nil {
return fmt.Errorf("can't create N3 client: %w", err)
}
@ -157,7 +157,7 @@ func dumpSingleContainer(bw *io.BufBinWriter, ch util.Uint160, inv *invoker.Invo
}
func listContainers(cmd *cobra.Command, _ []string) error {
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
if err != nil {
return fmt.Errorf("can't create N3 client: %w", err)
}

View file

@ -36,7 +36,7 @@ type contractDumpInfo struct {
}
func dumpContractHashes(cmd *cobra.Command, _ []string) error {
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
if err != nil {
return fmt.Errorf("can't create N3 client: %w", err)
}

View file

@ -0,0 +1,83 @@
package frostfsid
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
var (
frostfsidAddSubjectKeyCmd = &cobra.Command{
Use: "add-subject-key",
Short: "Add a public key to the subject in frostfsid contract",
PreRun: func(cmd *cobra.Command, _ []string) {
_ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag))
_ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag))
},
Run: frostfsidAddSubjectKey,
}
frostfsidRemoveSubjectKeyCmd = &cobra.Command{
Use: "remove-subject-key",
Short: "Remove a public key from the subject in frostfsid contract",
PreRun: func(cmd *cobra.Command, _ []string) {
_ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag))
_ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag))
},
Run: frostfsidRemoveSubjectKey,
}
)
func initFrostfsIDAddSubjectKeyCmd() {
Cmd.AddCommand(frostfsidAddSubjectKeyCmd)
ff := frostfsidAddSubjectKeyCmd.Flags()
ff.StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
ff.String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc)
ff.String(subjectAddressFlag, "", "Subject address")
_ = frostfsidAddSubjectKeyCmd.MarkFlagRequired(subjectAddressFlag)
ff.String(subjectKeyFlag, "", "Public key to add")
_ = frostfsidAddSubjectKeyCmd.MarkFlagRequired(subjectKeyFlag)
}
func initFrostfsIDRemoveSubjectKeyCmd() {
Cmd.AddCommand(frostfsidRemoveSubjectKeyCmd)
ff := frostfsidRemoveSubjectKeyCmd.Flags()
ff.StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
ff.String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc)
ff.String(subjectAddressFlag, "", "Subject address")
_ = frostfsidAddSubjectKeyCmd.MarkFlagRequired(subjectAddressFlag)
ff.String(subjectKeyFlag, "", "Public key to remove")
_ = frostfsidAddSubjectKeyCmd.MarkFlagRequired(subjectKeyFlag)
}
func frostfsidAddSubjectKey(cmd *cobra.Command, _ []string) {
addr := getFrostfsIDSubjectAddress(cmd)
pub := getFrostfsIDSubjectKey(cmd)
ffsid, err := newFrostfsIDClient(cmd)
commonCmd.ExitOnErr(cmd, "init contract client: %w", err)
ffsid.addCall(ffsid.roCli.AddSubjectKeyCall(addr, pub))
err = ffsid.sendWait()
commonCmd.ExitOnErr(cmd, "add subject key: %w", err)
}
func frostfsidRemoveSubjectKey(cmd *cobra.Command, _ []string) {
addr := getFrostfsIDSubjectAddress(cmd)
pub := getFrostfsIDSubjectKey(cmd)
ffsid, err := newFrostfsIDClient(cmd)
commonCmd.ExitOnErr(cmd, "init contract client: %w", err)
ffsid.addCall(ffsid.roCli.RemoveSubjectKeyCall(addr, pub))
err = ffsid.sendWait()
commonCmd.ExitOnErr(cmd, "remove subject key: %w", err)
}

View file

@ -1,7 +1,6 @@
package frostfsid
import (
"errors"
"fmt"
"math/big"
"sort"
@ -61,7 +60,6 @@ var (
Use: "list-namespaces",
Short: "List all namespaces in frostfsid",
PreRun: func(cmd *cobra.Command, _ []string) {
_ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag))
_ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag))
},
Run: frostfsidListNamespaces,
@ -91,7 +89,6 @@ var (
Use: "list-subjects",
Short: "List subjects in namespace",
PreRun: func(cmd *cobra.Command, _ []string) {
_ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag))
_ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag))
},
Run: frostfsidListSubjects,
@ -121,7 +118,6 @@ var (
Use: "list-groups",
Short: "List groups in namespace",
PreRun: func(cmd *cobra.Command, _ []string) {
_ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag))
_ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag))
},
Run: frostfsidListGroups,
@ -151,7 +147,6 @@ var (
Use: "list-group-subjects",
Short: "List subjects in group",
PreRun: func(cmd *cobra.Command, _ []string) {
_ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag))
_ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag))
},
Run: frostfsidListGroupSubjects,
@ -169,7 +164,6 @@ func initFrostfsIDCreateNamespaceCmd() {
func initFrostfsIDListNamespacesCmd() {
Cmd.AddCommand(frostfsidListNamespacesCmd)
frostfsidListNamespacesCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
frostfsidListNamespacesCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc)
}
func initFrostfsIDCreateSubjectCmd() {
@ -193,7 +187,6 @@ func initFrostfsIDListSubjectsCmd() {
frostfsidListSubjectsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
frostfsidListSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace to list subjects")
frostfsidListSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)")
frostfsidListSubjectsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc)
}
func initFrostfsIDCreateGroupCmd() {
@ -217,7 +210,6 @@ func initFrostfsIDListGroupsCmd() {
Cmd.AddCommand(frostfsidListGroupsCmd)
frostfsidListGroupsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
frostfsidListGroupsCmd.Flags().String(namespaceFlag, "", "Namespace to list groups")
frostfsidListGroupsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc)
}
func initFrostfsIDAddSubjectToGroupCmd() {
@ -242,7 +234,6 @@ func initFrostfsIDListGroupSubjectsCmd() {
frostfsidListGroupSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace name")
frostfsidListGroupSubjectsCmd.Flags().Int64(groupIDFlag, 0, "Group id")
frostfsidListGroupSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)")
frostfsidListGroupSubjectsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc)
}
func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) {
@ -497,10 +488,6 @@ func (f *frostfsidClient) sendWaitRes() (*state.AppExecResult, error) {
}
f.bw.Reset()
if len(f.wCtx.SentTxs) == 0 {
return nil, errors.New("no transactions to wait")
}
f.wCtx.Command.Println("Waiting for transactions to persist...")
return f.roCli.Wait(f.wCtx.SentTxs[0].Hash, f.wCtx.SentTxs[0].Vub, nil)
}
@ -522,7 +509,7 @@ func readIterator(inv *invoker.Invoker, iter *result.Iterator, batchSize int, se
}
func initInvoker(cmd *cobra.Command) (*invoker.Invoker, *state.Contract, util.Uint160) {
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
commonCmd.ExitOnErr(cmd, "can't create N3 client: %w", err)
inv := invoker.New(c, nil)

View file

@ -1,59 +1,12 @@
package frostfsid
import (
"encoding/hex"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/ape"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/encoding/address"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
func TestFrostfsIDConfig(t *testing.T) {
pks := make([]*keys.PrivateKey, 4)
for i := range pks {
pk, err := keys.NewPrivateKey()
require.NoError(t, err)
pks[i] = pk
}
fmts := []string{
pks[0].GetScriptHash().StringLE(),
address.Uint160ToString(pks[1].GetScriptHash()),
hex.EncodeToString(pks[2].PublicKey().UncompressedBytes()),
hex.EncodeToString(pks[3].PublicKey().Bytes()),
}
for i := range fmts {
v := viper.New()
v.Set("frostfsid.admin", fmts[i])
actual, found, err := helper.GetFrostfsIDAdmin(v)
require.NoError(t, err)
require.True(t, found)
require.Equal(t, pks[i].GetScriptHash(), actual)
}
t.Run("bad key", func(t *testing.T) {
v := viper.New()
v.Set("frostfsid.admin", "abc")
_, found, err := helper.GetFrostfsIDAdmin(v)
require.Error(t, err)
require.True(t, found)
})
t.Run("missing key", func(t *testing.T) {
v := viper.New()
_, found, err := helper.GetFrostfsIDAdmin(v)
require.NoError(t, err)
require.False(t, found)
})
}
func TestNamespaceRegexp(t *testing.T) {
for _, tc := range []struct {
name string

View file

@ -12,4 +12,6 @@ func init() {
initFrostfsIDAddSubjectToGroupCmd()
initFrostfsIDRemoveSubjectFromGroupCmd()
initFrostfsIDListGroupSubjectsCmd()
initFrostfsIDAddSubjectKeyCmd()
initFrostfsIDRemoveSubjectKeyCmd()
}

View file

@ -5,7 +5,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
"github.com/google/uuid"
"github.com/nspcc-dev/neo-go/pkg/core/state"
@ -31,42 +30,32 @@ type LocalActor struct {
// NewLocalActor create LocalActor with accounts form provided wallets.
// In case of empty wallets provided created actor with dummy account only for read operation.
func NewLocalActor(cmd *cobra.Command, c actor.RPCActor) (*LocalActor, error) {
//
// If wallets are provided, the contract client will use accounts with accName name from these wallets.
// To determine which account name should be used in a contract client, refer to how the contract
// verifies the transaction signature.
func NewLocalActor(cmd *cobra.Command, c actor.RPCActor, accName string) (*LocalActor, error) {
walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag))
var act *actor.Actor
var accounts []*wallet.Account
if walletDir == "" {
account, err := wallet.NewAccount()
commonCmd.ExitOnErr(cmd, "unable to create dummy account: %w", err)
act, err = actor.New(c, []actor.SignerAccount{{
Signer: transaction.Signer{
Account: account.Contract.ScriptHash(),
Scopes: transaction.Global,
},
Account: account,
}})
if err != nil {
return nil, err
}
} else {
wallets, err := GetAlphabetWallets(viper.GetViper(), walletDir)
commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err)
for _, w := range wallets {
acc, err := GetWalletAccount(w, constants.CommitteeAccountName)
commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err)
accounts = append(accounts, acc)
}
act, err = actor.New(c, []actor.SignerAccount{{
Signer: transaction.Signer{
Account: accounts[0].Contract.ScriptHash(),
Scopes: transaction.Global,
},
Account: accounts[0],
}})
if err != nil {
return nil, err
}
wallets, err := GetAlphabetWallets(viper.GetViper(), walletDir)
commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err)
for _, w := range wallets {
acc, err := GetWalletAccount(w, accName)
commonCmd.ExitOnErr(cmd, fmt.Sprintf("can't find %s account: %%w", accName), err)
accounts = append(accounts, acc)
}
act, err = actor.New(c, []actor.SignerAccount{{
Signer: transaction.Signer{
Account: accounts[0].Contract.ScriptHash(),
Scopes: transaction.Global,
},
Account: accounts[0],
}})
if err != nil {
return nil, err
}
return &LocalActor{
neoActor: act,

View file

@ -82,7 +82,7 @@ func GetContractDeployData(c *InitializeContext, ctrName string, keysParam []any
h, found, err = getFrostfsIDAdminFromContract(c.ReadOnlyInvoker)
}
if method != constants.UpdateMethodName || err == nil && !found {
h, found, err = GetFrostfsIDAdmin(viper.GetViper())
h, found, err = getFrostfsIDAdmin(viper.GetViper())
}
if err != nil {
return nil, err

View file

@ -11,7 +11,7 @@ import (
const frostfsIDAdminConfigKey = "frostfsid.admin"
func GetFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) {
func getFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) {
admin := v.GetString(frostfsIDAdminConfigKey)
if admin == "" {
return util.Uint160{}, false, nil

View file

@ -0,0 +1,53 @@
package helper
import (
"encoding/hex"
"testing"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/encoding/address"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
func TestFrostfsIDConfig(t *testing.T) {
pks := make([]*keys.PrivateKey, 4)
for i := range pks {
pk, err := keys.NewPrivateKey()
require.NoError(t, err)
pks[i] = pk
}
fmts := []string{
pks[0].GetScriptHash().StringLE(),
address.Uint160ToString(pks[1].GetScriptHash()),
hex.EncodeToString(pks[2].PublicKey().UncompressedBytes()),
hex.EncodeToString(pks[3].PublicKey().Bytes()),
}
for i := range fmts {
v := viper.New()
v.Set("frostfsid.admin", fmts[i])
actual, found, err := getFrostfsIDAdmin(v)
require.NoError(t, err)
require.True(t, found)
require.Equal(t, pks[i].GetScriptHash(), actual)
}
t.Run("bad key", func(t *testing.T) {
v := viper.New()
v.Set("frostfsid.admin", "abc")
_, found, err := getFrostfsIDAdmin(v)
require.Error(t, err)
require.True(t, found)
})
t.Run("missing key", func(t *testing.T) {
v := viper.New()
_, found, err := getFrostfsIDAdmin(v)
require.NoError(t, err)
require.False(t, found)
})
}

View file

@ -134,12 +134,12 @@ func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContex
return nil, err
}
accounts, err := createWalletAccounts(wallets)
accounts, err := getSingleAccounts(wallets)
if err != nil {
return nil, err
}
cliCtx, err := DefaultClientContext(c, committeeAcc)
cliCtx, err := defaultClientContext(c, committeeAcc)
if err != nil {
return nil, fmt.Errorf("client context: %w", err)
}
@ -191,7 +191,7 @@ func createClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet)
}
c, err = NewLocalClient(cmd, v, wallets, ldf.Value.String())
} else {
c, err = GetN3Client(v)
c, err = NewRemoteClient(v)
}
if err != nil {
return nil, fmt.Errorf("can't create N3 client: %w", err)
@ -211,7 +211,7 @@ func getContractsPath(cmd *cobra.Command, needContracts bool) (string, error) {
return ctrPath, nil
}
func createWalletAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) {
func getSingleAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) {
accounts := make([]*wallet.Account, len(wallets))
for i, w := range wallets {
acc, err := GetWalletAccount(w, constants.SingleAccountName)

View file

@ -8,6 +8,7 @@ import (
"sort"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants"
"github.com/google/uuid"
"github.com/nspcc-dev/neo-go/pkg/config"
@ -47,7 +48,7 @@ type LocalClient struct {
}
func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet, dumpPath string) (*LocalClient, error) {
cfg, err := config.LoadFile(v.GetString(constants.ProtoConfigPath))
cfg, err := config.LoadFile(v.GetString(commonflags.ProtoConfigPath))
if err != nil {
return nil, err
}
@ -57,17 +58,59 @@ func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet
return nil, err
}
m := smartcontract.GetDefaultHonestNodeCount(int(cfg.ProtocolConfiguration.ValidatorsCount))
accounts := make([]*wallet.Account, len(wallets))
for i := range accounts {
accounts[i], err = GetWalletAccount(wallets[i], constants.ConsensusAccountName)
if err != nil {
return nil, err
go bc.Run()
accounts, err := getBlockSigningAccounts(cfg.ProtocolConfiguration, wallets)
if err != nil {
return nil, err
}
if cmd.Name() != "init" {
if err := restoreDump(bc, dumpPath); err != nil {
return nil, fmt.Errorf("restore dump: %w", err)
}
}
return &LocalClient{
bc: bc,
dumpPath: dumpPath,
accounts: accounts,
}, nil
}
func restoreDump(bc *core.Blockchain, dumpPath string) error {
f, err := os.OpenFile(dumpPath, os.O_RDONLY, 0o600)
if err != nil {
return fmt.Errorf("can't open local dump: %w", err)
}
defer f.Close()
r := io.NewBinReaderFromIO(f)
var skip uint32
if bc.BlockHeight() != 0 {
skip = bc.BlockHeight() + 1
}
count := r.ReadU32LE() - skip
if err := chaindump.Restore(bc, r, skip, count, nil); err != nil {
return err
}
return nil
}
func getBlockSigningAccounts(cfg config.ProtocolConfiguration, wallets []*wallet.Wallet) ([]*wallet.Account, error) {
accounts := make([]*wallet.Account, len(wallets))
for i := range accounts {
acc, err := GetWalletAccount(wallets[i], constants.ConsensusAccountName)
if err != nil {
return nil, err
}
accounts[i] = acc
}
indexMap := make(map[string]int)
for i, pub := range cfg.ProtocolConfiguration.StandbyCommittee {
for i, pub := range cfg.StandbyCommittee {
indexMap[pub] = i
}
@ -76,37 +119,12 @@ func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet
pj := accounts[j].PrivateKey().PublicKey().Bytes()
return indexMap[string(pi)] < indexMap[string(pj)]
})
sort.Slice(accounts[:cfg.ProtocolConfiguration.ValidatorsCount], func(i, j int) bool {
sort.Slice(accounts[:cfg.ValidatorsCount], func(i, j int) bool {
return accounts[i].PublicKey().Cmp(accounts[j].PublicKey()) == -1
})
go bc.Run()
if cmd.Name() != "init" {
f, err := os.OpenFile(dumpPath, os.O_RDONLY, 0o600)
if err != nil {
return nil, fmt.Errorf("can't open local dump: %w", err)
}
defer f.Close()
r := io.NewBinReaderFromIO(f)
var skip uint32
if bc.BlockHeight() != 0 {
skip = bc.BlockHeight() + 1
}
count := r.ReadU32LE() - skip
if err := chaindump.Restore(bc, r, skip, count, nil); err != nil {
return nil, fmt.Errorf("can't restore local dump: %w", err)
}
}
return &LocalClient{
bc: bc,
dumpPath: dumpPath,
accounts: accounts[:m],
}, nil
m := smartcontract.GetDefaultHonestNodeCount(int(cfg.ValidatorsCount))
return accounts[:m], nil
}
func (l *LocalClient) GetBlockCount() (uint32, error) {
@ -127,11 +145,6 @@ func (l *LocalClient) GetApplicationLog(h util.Uint256, t *trigger.Type) (*resul
return &a, nil
}
func (l *LocalClient) GetCommittee() (keys.PublicKeys, error) {
// not used by `morph init` command
panic("unexpected call")
}
// InvokeFunction is implemented via `InvokeScript`.
func (l *LocalClient) InvokeFunction(h util.Uint160, method string, sPrm []smartcontract.Parameter, ss []transaction.Signer) (*result.Invoke, error) {
var err error
@ -295,13 +308,7 @@ func (l *LocalClient) InvokeScript(script []byte, signers []transaction.Signer)
}
func (l *LocalClient) SendRawTransaction(tx *transaction.Transaction) (util.Uint256, error) {
// We need to test that transaction was formed correctly to catch as many errors as we can.
bs := tx.Bytes()
_, err := transaction.NewTransactionFromBytes(bs)
if err != nil {
return tx.Hash(), fmt.Errorf("invalid transaction: %w", err)
}
tx = tx.Copy()
l.transactions = append(l.transactions, tx)
return tx.Hash(), nil
}

View file

@ -10,7 +10,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags"
"github.com/nspcc-dev/neo-go/pkg/core/state"
"github.com/nspcc-dev/neo-go/pkg/core/transaction"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/neorpc/result"
"github.com/nspcc-dev/neo-go/pkg/rpcclient"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/actor"
@ -25,15 +24,10 @@ import (
// Client represents N3 client interface capable of test-invoking scripts
// and sending signed transactions to chain.
type Client interface {
invoker.RPCInvoke
actor.RPCActor
GetBlockCount() (uint32, error)
GetNativeContracts() ([]state.Contract, error)
GetApplicationLog(util.Uint256, *trigger.Type) (*result.ApplicationLog, error)
GetVersion() (*result.Version, error)
SendRawTransaction(*transaction.Transaction) (util.Uint256, error)
GetCommittee() (keys.PublicKeys, error)
CalculateNetworkFee(tx *transaction.Transaction) (int64, error)
}
type HashVUBPair struct {
@ -48,7 +42,7 @@ type ClientContext struct {
SentTxs []HashVUBPair
}
func GetN3Client(v *viper.Viper) (Client, error) {
func NewRemoteClient(v *viper.Viper) (Client, error) {
// number of opened connections
// by neo-go client per one host
const (
@ -88,8 +82,14 @@ func GetN3Client(v *viper.Viper) (Client, error) {
return c, nil
}
func DefaultClientContext(c Client, committeeAcc *wallet.Account) (*ClientContext, error) {
commAct, err := NewActor(c, committeeAcc)
func defaultClientContext(c Client, committeeAcc *wallet.Account) (*ClientContext, error) {
commAct, err := actor.New(c, []actor.SignerAccount{{
Signer: transaction.Signer{
Account: committeeAcc.Contract.ScriptHash(),
Scopes: transaction.Global,
},
Account: committeeAcc,
}})
if err != nil {
return nil, err
}

View file

@ -15,10 +15,8 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring"
"github.com/nspcc-dev/neo-go/pkg/core/state"
"github.com/nspcc-dev/neo-go/pkg/core/transaction"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/encoding/fixedn"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/actor"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/management"
"github.com/nspcc-dev/neo-go/pkg/wallet"
"github.com/spf13/viper"
@ -87,16 +85,6 @@ func openAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, er
return wallets, nil
}
func NewActor(c actor.RPCActor, committeeAcc *wallet.Account) (*actor.Actor, error) {
return actor.New(c, []actor.SignerAccount{{
Signer: transaction.Signer{
Account: committeeAcc.Contract.ScriptHash(),
Scopes: transaction.Global,
},
Account: committeeAcc,
}})
}
func ReadContract(ctrPath, ctrName string) (*ContractState, error) {
rawNef, err := os.ReadFile(filepath.Join(ctrPath, ctrName+"_contract.nef"))
if err != nil {

View file

@ -62,7 +62,7 @@ func testInitialize(t *testing.T, committeeSize int) {
v := viper.GetViper()
require.NoError(t, generateTestData(testdataDir, committeeSize))
v.Set(constants.ProtoConfigPath, filepath.Join(testdataDir, protoFileName))
v.Set(commonflags.ProtoConfigPath, filepath.Join(testdataDir, protoFileName))
// Set to the path or remove the next statement to download from the network.
require.NoError(t, Cmd.Flags().Set(commonflags.ContractsInitFlag, contractsPath))

View file

@ -2,7 +2,6 @@ package initialize
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
@ -32,7 +31,7 @@ var Cmd = &cobra.Command{
_ = viper.BindPFlag(commonflags.ContainerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag))
_ = viper.BindPFlag(commonflags.ContainerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag))
_ = viper.BindPFlag(commonflags.WithdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag))
_ = viper.BindPFlag(constants.ProtoConfigPath, cmd.Flags().Lookup(constants.ProtoConfigPath))
_ = viper.BindPFlag(commonflags.ProtoConfigPath, cmd.Flags().Lookup(commonflags.ProtoConfigPath))
},
RunE: initializeSideChainCmd,
}
@ -48,7 +47,7 @@ func initInitCmd() {
// Defaults are taken from neo-preodolenie.
Cmd.Flags().Uint64(containerFeeCLIFlag, 1000, "Container registration fee")
Cmd.Flags().Uint64(containerAliasFeeCLIFlag, 500, "Container alias fee")
Cmd.Flags().String(constants.ProtoConfigPath, "", "Path to the consensus node configuration")
Cmd.Flags().String(commonflags.ProtoConfigPath, "", "Path to the consensus node configuration")
Cmd.Flags().String(commonflags.LocalDumpFlag, "", "Path to the blocks dump file")
Cmd.MarkFlagsMutuallyExclusive(commonflags.ContractsInitFlag, commonflags.ContractsURLFlag)
}

View file

@ -13,7 +13,7 @@ import (
)
func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) {
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
commonCmd.ExitOnErr(cmd, "can't create N3 client: %w", err)
inv := invoker.New(c, nil)

View file

@ -12,7 +12,6 @@ var (
Short: "List netmap candidates nodes",
PreRun: func(cmd *cobra.Command, _ []string) {
_ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag))
_ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag))
},
Run: listNetmapCandidatesNodes,
}

View file

@ -24,7 +24,7 @@ func initRegisterCmd() {
}
func registerDomain(cmd *cobra.Command, _ []string) {
c, actor, _ := getRPCClient(cmd)
c, actor := nnsWriter(cmd)
name, _ := cmd.Flags().GetString(nnsNameFlag)
email, _ := cmd.Flags().GetString(nnsEmailFlag)
@ -53,7 +53,7 @@ func initDeleteCmd() {
}
func deleteDomain(cmd *cobra.Command, _ []string) {
c, actor, _ := getRPCClient(cmd)
c, actor := nnsWriter(cmd)
name, _ := cmd.Flags().GetString(nnsNameFlag)
h, vub, err := c.DeleteDomain(name)

View file

@ -2,24 +2,37 @@ package nns
import (
client "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/nns"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/management"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
func getRPCClient(cmd *cobra.Command) (*client.Contract, *helper.LocalActor, util.Uint160) {
func nnsWriter(cmd *cobra.Command) (*client.Contract, *helper.LocalActor) {
v := viper.GetViper()
c, err := helper.GetN3Client(v)
c, err := helper.NewRemoteClient(v)
commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err)
ac, err := helper.NewLocalActor(cmd, c)
ac, err := helper.NewLocalActor(cmd, c, constants.CommitteeAccountName)
commonCmd.ExitOnErr(cmd, "can't create actor: %w", err)
r := management.NewReader(ac.Invoker)
nnsCs, err := helper.GetContractByID(r, 1)
commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err)
return client.New(ac, nnsCs.Hash), ac, nnsCs.Hash
return client.New(ac, nnsCs.Hash), ac
}
func nnsReader(cmd *cobra.Command) (*client.ContractReader, *invoker.Invoker) {
c, err := helper.NewRemoteClient(viper.GetViper())
commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err)
inv := invoker.New(c, nil)
r := management.NewReader(inv)
nnsCs, err := helper.GetContractByID(r, 1)
commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err)
return client.NewReader(inv, nnsCs.Hash), inv
}

View file

@ -8,7 +8,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-contract/nns"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap"
"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
"github.com/spf13/cobra"
)
@ -29,7 +28,6 @@ func initAddRecordCmd() {
func initGetRecordsCmd() {
Cmd.AddCommand(getRecordsCmd)
getRecordsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
getRecordsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc)
getRecordsCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc)
getRecordsCmd.Flags().String(nnsRecordTypeFlag, "", nnsRecordTypeFlagDesc)
@ -61,7 +59,7 @@ func initDelRecordCmd() {
}
func addRecord(cmd *cobra.Command, _ []string) {
c, actor, _ := getRPCClient(cmd)
c, actor := nnsWriter(cmd)
name, _ := cmd.Flags().GetString(nnsNameFlag)
data, _ := cmd.Flags().GetString(nnsRecordDataFlag)
recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag)
@ -77,16 +75,16 @@ func addRecord(cmd *cobra.Command, _ []string) {
}
func getRecords(cmd *cobra.Command, _ []string) {
c, act, hash := getRPCClient(cmd)
c, inv := nnsReader(cmd)
name, _ := cmd.Flags().GetString(nnsNameFlag)
recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag)
if recordType == "" {
sid, r, err := unwrap.SessionIterator(act.Invoker.Call(hash, "getAllRecords", name))
sid, r, err := c.GetAllRecords(name)
commonCmd.ExitOnErr(cmd, "unable to get records: %w", err)
defer func() {
_ = act.Invoker.TerminateSession(sid)
_ = inv.TerminateSession(sid)
}()
items, err := act.Invoker.TraverseIterator(sid, &r, 0)
items, err := inv.TraverseIterator(sid, &r, 0)
commonCmd.ExitOnErr(cmd, "unable to get records: %w", err)
for len(items) != 0 {
for j := range items {
@ -97,7 +95,7 @@ func getRecords(cmd *cobra.Command, _ []string) {
recordTypeToString(nns.RecordType(rs[1].Value().(*big.Int).Int64())),
string(bs))
}
items, err = act.Invoker.TraverseIterator(sid, &r, 0)
items, err = inv.TraverseIterator(sid, &r, 0)
commonCmd.ExitOnErr(cmd, "unable to get records: %w", err)
}
} else {
@ -114,7 +112,7 @@ func getRecords(cmd *cobra.Command, _ []string) {
}
func delRecords(cmd *cobra.Command, _ []string) {
c, actor, _ := getRPCClient(cmd)
c, actor := nnsWriter(cmd)
name, _ := cmd.Flags().GetString(nnsNameFlag)
recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag)
typ, err := getRecordType(recordType)
@ -129,7 +127,7 @@ func delRecords(cmd *cobra.Command, _ []string) {
}
func delRecord(cmd *cobra.Command, _ []string) {
c, actor, _ := getRPCClient(cmd)
c, actor := nnsWriter(cmd)
name, _ := cmd.Flags().GetString(nnsNameFlag)
data, _ := cmd.Flags().GetString(nnsRecordDataFlag)
recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag)

View file

@ -14,7 +14,7 @@ func initRenewCmd() {
}
func renewDomain(cmd *cobra.Command, _ []string) {
c, actor, _ := getRPCClient(cmd)
c, actor := nnsWriter(cmd)
name, _ := cmd.Flags().GetString(nnsNameFlag)
h, vub, err := c.Renew(name)
commonCmd.ExitOnErr(cmd, "unable to renew domain: %w", err)

View file

@ -18,12 +18,11 @@ const (
func initTokensCmd() {
Cmd.AddCommand(tokensCmd)
tokensCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc)
tokensCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc)
tokensCmd.Flags().BoolP(commonflags.Verbose, commonflags.VerboseShorthand, false, verboseDesc)
}
func listTokens(cmd *cobra.Command, _ []string) {
c, _, _ := getRPCClient(cmd)
c, _ := nnsReader(cmd)
it, err := c.Tokens()
commonCmd.ExitOnErr(cmd, "unable to get tokens: %w", err)
for toks, err := it.Next(10); err == nil && len(toks) > 0; toks, err = it.Next(10) {
@ -41,7 +40,7 @@ func listTokens(cmd *cobra.Command, _ []string) {
}
}
func getCnameRecord(c *client.Contract, token []byte) (string, error) {
func getCnameRecord(c *client.ContractReader, token []byte) (string, error) {
items, err := c.GetRecords(string(token), big.NewInt(int64(nns.CNAME)))
// GetRecords returns the error "not an array" if the domain does not contain records.

View file

@ -30,7 +30,7 @@ func initUpdateCmd() {
}
func updateSOA(cmd *cobra.Command, _ []string) {
c, actor, _ := getRPCClient(cmd)
c, actor := nnsWriter(cmd)
name, _ := cmd.Flags().GetString(nnsNameFlag)
email, _ := cmd.Flags().GetString(nnsEmailFlag)

View file

@ -89,7 +89,7 @@ func depositNotary(cmd *cobra.Command, _ []string) error {
}
func transferGas(cmd *cobra.Command, acc *wallet.Account, accHash util.Uint160, gasAmount fixedn.Fixed8, till int64) error {
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
if err != nil {
return err
}

View file

@ -62,7 +62,7 @@ func SetPolicyCmd(cmd *cobra.Command, args []string) error {
}
func dumpPolicyCmd(cmd *cobra.Command, _ []string) error {
c, err := helper.GetN3Client(viper.GetViper())
c, err := helper.NewRemoteClient(viper.GetViper())
commonCmd.ExitOnErr(cmd, "can't create N3 client:", err)
inv := invoker.New(c, nil)

View file

@ -72,4 +72,3 @@ All other `object` sub-commands support only static sessions (2).
List of commands supporting sessions (static only):
- `create`
- `delete`
- `set-eacl`

View file

@ -9,7 +9,6 @@ import (
"io"
"os"
"slices"
"sort"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/accounting"
@ -78,13 +77,31 @@ func ListContainers(ctx context.Context, prm ListContainersPrm) (res ListContain
// SortedIDList returns sorted list of identifiers of user's containers.
func (x ListContainersRes) SortedIDList() []cid.ID {
list := x.cliRes.Containers()
sort.Slice(list, func(i, j int) bool {
lhs, rhs := list[i].EncodeToString(), list[j].EncodeToString()
return strings.Compare(lhs, rhs) < 0
slices.SortFunc(list, func(lhs, rhs cid.ID) int {
return strings.Compare(lhs.EncodeToString(), rhs.EncodeToString())
})
return list
}
func ListContainersStream(ctx context.Context, prm ListContainersPrm, processCnr func(id cid.ID) bool) (err error) {
cliPrm := &client.PrmContainerListStream{
XHeaders: prm.XHeaders,
OwnerID: prm.OwnerID,
Session: prm.Session,
}
rdr, err := prm.cli.ContainerListInit(ctx, *cliPrm)
if err != nil {
return fmt.Errorf("init container list: %w", err)
}
err = rdr.Iterate(processCnr)
if err != nil {
return fmt.Errorf("read container list: %w", err)
}
return
}
// PutContainerPrm groups parameters of PutContainer operation.
type PutContainerPrm struct {
Client *client.Client

View file

@ -1,44 +1,19 @@
package apemanager
import (
"encoding/hex"
"errors"
"fmt"
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape"
client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
"github.com/spf13/cobra"
)
const (
chainIDFlag = "chain-id"
chainIDHexFlag = "chain-id-hex"
ruleFlag = "rule"
pathFlag = "path"
)
const (
targetNameFlag = "target-name"
targetNameDesc = "Resource name in APE resource name format"
targetTypeFlag = "target-type"
targetTypeDesc = "Resource type(container/namespace)"
)
const (
namespaceTarget = "namespace"
containerTarget = "container"
userTarget = "user"
groupTarget = "group"
)
var errUnknownTargetType = errors.New("unknown target type")
var addCmd = &cobra.Command{
Use: "add",
Short: "Add rule chain for a target",
@ -49,55 +24,28 @@ var addCmd = &cobra.Command{
}
func parseTarget(cmd *cobra.Command) (ct apeSDK.ChainTarget) {
typ, _ := cmd.Flags().GetString(targetTypeFlag)
name, _ := cmd.Flags().GetString(targetNameFlag)
t := apeCmd.ParseTarget(cmd)
ct.Name = name
ct.Name = t.Name
switch typ {
case namespaceTarget:
switch t.Type {
case engine.Namespace:
ct.TargetType = apeSDK.TargetTypeNamespace
case containerTarget:
var cnr cid.ID
commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(name))
case engine.Container:
ct.TargetType = apeSDK.TargetTypeContainer
case userTarget:
case engine.User:
ct.TargetType = apeSDK.TargetTypeUser
case groupTarget:
case engine.Group:
ct.TargetType = apeSDK.TargetTypeGroup
default:
commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType)
commonCmd.ExitOnErr(cmd, "conversion error: %w", fmt.Errorf("unknown type '%c'", t.Type))
}
return ct
}
func parseChain(cmd *cobra.Command) apeSDK.Chain {
chainID, _ := cmd.Flags().GetString(chainIDFlag)
hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag)
chainIDRaw := []byte(chainID)
if hexEncoded {
var err error
chainIDRaw, err = hex.DecodeString(chainID)
commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err)
}
chain := new(apechain.Chain)
chain.ID = apechain.ID(chainIDRaw)
if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 {
commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, rules))
} else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" {
commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", util.ParseAPEChainBinaryOrJSON(chain, encPath))
} else {
commonCmd.ExitOnErr(cmd, "parser error: %w", errors.New("rule is not passed"))
}
cmd.Println("Parsed chain:")
util.PrintHumanReadableAPEChain(cmd, chain)
serialized := chain.Bytes()
c := apeCmd.ParseChain(cmd)
serialized := c.Bytes()
return apeSDK.Chain{
Raw: serialized,
}
@ -126,13 +74,13 @@ func initAddCmd() {
commonflags.Init(addCmd)
ff := addCmd.Flags()
ff.StringArray(ruleFlag, []string{}, "Rule statement")
ff.String(pathFlag, "", "Path to encoded chain in JSON or binary format")
ff.String(chainIDFlag, "", "Assign ID to the parsed chain")
ff.String(targetNameFlag, "", targetNameDesc)
ff.String(targetTypeFlag, "", targetTypeDesc)
_ = addCmd.MarkFlagRequired(targetTypeFlag)
ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex")
ff.StringArray(apeCmd.RuleFlag, []string{}, apeCmd.RuleFlagDesc)
ff.String(apeCmd.PathFlag, "", apeCmd.PathFlagDesc)
ff.String(apeCmd.ChainIDFlag, "", apeCmd.ChainIDFlagDesc)
ff.String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc)
ff.String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc)
_ = addCmd.MarkFlagRequired(apeCmd.TargetTypeFlag)
ff.Bool(apeCmd.ChainIDHexFlag, false, apeCmd.ChainIDHexFlagDesc)
addCmd.MarkFlagsMutuallyExclusive(pathFlag, ruleFlag)
addCmd.MarkFlagsMutuallyExclusive(apeCmd.PathFlag, apeCmd.RuleFlag)
}

View file

@ -4,8 +4,8 @@ import (
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
apeutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"github.com/spf13/cobra"
@ -35,7 +35,7 @@ func list(cmd *cobra.Command, _ []string) {
for _, respChain := range resp.Chains {
var chain apechain.Chain
commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(respChain.Raw))
apeutil.PrintHumanReadableAPEChain(cmd, &chain)
apeCmd.PrintHumanReadableAPEChain(cmd, &chain)
}
}
@ -43,7 +43,7 @@ func initListCmd() {
commonflags.Init(listCmd)
ff := listCmd.Flags()
ff.String(targetNameFlag, "", targetNameDesc)
ff.String(targetTypeFlag, "", targetTypeDesc)
_ = listCmd.MarkFlagRequired(targetTypeFlag)
ff.String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc)
ff.String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc)
_ = listCmd.MarkFlagRequired(apeCmd.TargetTypeFlag)
}

View file

@ -1,29 +1,23 @@
package apemanager
import (
"encoding/hex"
"errors"
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"github.com/spf13/cobra"
)
var (
errEmptyChainID = errors.New("chain id cannot be empty")
removeCmd = &cobra.Command{
Use: "remove",
Short: "Remove rule chain for a target",
Run: remove,
PersistentPreRun: func(cmd *cobra.Command, _ []string) {
commonflags.Bind(cmd)
},
}
)
var removeCmd = &cobra.Command{
Use: "remove",
Short: "Remove rule chain for a target",
Run: remove,
PersistentPreRun: func(cmd *cobra.Command, _ []string) {
commonflags.Bind(cmd)
},
}
func remove(cmd *cobra.Command, _ []string) {
target := parseTarget(cmd)
@ -31,19 +25,9 @@ func remove(cmd *cobra.Command, _ []string) {
key := key.Get(cmd)
cli := internalclient.GetSDKClientByFlag(cmd, key, commonflags.RPC)
chainID, _ := cmd.Flags().GetString(chainIDFlag)
if chainID == "" {
commonCmd.ExitOnErr(cmd, "read chain id error: %w", errEmptyChainID)
}
chainID := apeCmd.ParseChainID(cmd)
chainIDRaw := []byte(chainID)
hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag)
if hexEncoded {
var err error
chainIDRaw, err = hex.DecodeString(chainID)
commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err)
}
_, err := cli.APEManagerRemoveChain(cmd.Context(), client_sdk.PrmAPEManagerRemoveChain{
ChainTarget: target,
ChainID: chainIDRaw,
@ -58,9 +42,10 @@ func initRemoveCmd() {
commonflags.Init(removeCmd)
ff := removeCmd.Flags()
ff.String(targetNameFlag, "", targetNameDesc)
ff.String(targetTypeFlag, "", targetTypeDesc)
_ = removeCmd.MarkFlagRequired(targetTypeFlag)
ff.String(chainIDFlag, "", "Chain id")
ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex")
ff.String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc)
ff.String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc)
_ = removeCmd.MarkFlagRequired(apeCmd.TargetTypeFlag)
ff.String(apeCmd.ChainIDFlag, "", apeCmd.ChainIDFlagDesc)
_ = removeCmd.MarkFlagRequired(apeCmd.ChainIDFlag)
ff.Bool(apeCmd.ChainIDHexFlag, false, apeCmd.ChainIDHexFlagDesc)
}

View file

@ -1,31 +1,20 @@
package bearer
import (
"errors"
"fmt"
"os"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"github.com/spf13/cobra"
)
var (
errChainIDCannotBeEmpty = errors.New("chain id cannot be empty")
errRuleIsNotParsed = errors.New("rule is not passed")
)
const (
chainIDFlag = "chain-id"
chainIDHexFlag = "chain-id-hex"
ruleFlag = "rule"
pathFlag = "path"
outputFlag = "output"
outputFlag = "output"
)
var generateAPEOverrideCmd = &cobra.Command{
@ -40,7 +29,7 @@ Generated APE override can be dumped to a file in JSON format that is passed to
}
func genereateAPEOverride(cmd *cobra.Command, _ []string) {
c := parseChain(cmd)
c := apeCmd.ParseChain(cmd)
targetCID, _ := cmd.Flags().GetString(commonflags.CIDFlag)
var cid cidSDK.ID
@ -77,39 +66,11 @@ func init() {
ff.StringP(commonflags.CIDFlag, "", "", "Target container ID.")
_ = cobra.MarkFlagRequired(createCmd.Flags(), commonflags.CIDFlag)
ff.StringArray(ruleFlag, []string{}, "Rule statement")
ff.String(pathFlag, "", "Path to encoded chain in JSON or binary format")
ff.String(chainIDFlag, "", "Assign ID to the parsed chain")
ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex")
ff.StringArray(apeCmd.RuleFlag, []string{}, "Rule statement")
ff.String(apeCmd.PathFlag, "", "Path to encoded chain in JSON or binary format")
ff.String(apeCmd.ChainIDFlag, "", "Assign ID to the parsed chain")
ff.Bool(apeCmd.ChainIDHexFlag, false, "Flag to parse chain ID as hex")
ff.String(outputFlag, "", "Output path to dump result JSON-encoded APE override")
_ = cobra.MarkFlagFilename(createCmd.Flags(), outputFlag)
}
func parseChainID(cmd *cobra.Command) apechain.ID {
chainID, _ := cmd.Flags().GetString(chainIDFlag)
if chainID == "" {
commonCmd.ExitOnErr(cmd, "read chain id error: %w",
errChainIDCannotBeEmpty)
}
return apechain.ID(chainID)
}
func parseChain(cmd *cobra.Command) *apechain.Chain {
chain := new(apechain.Chain)
if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 {
commonCmd.ExitOnErr(cmd, "parser error: %w", parseutil.ParseAPEChain(chain, rules))
} else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" {
commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath))
} else {
commonCmd.ExitOnErr(cmd, "parser error: %w", errRuleIsNotParsed)
}
chain.ID = parseChainID(cmd)
cmd.Println("Parsed chain:")
parseutil.PrintHumanReadableAPEChain(cmd, chain)
return chain
}

View file

@ -15,14 +15,12 @@ import (
containerApi "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/spf13/cobra"
)
var (
containerACL string
containerPolicy string
containerAttributes []string
containerAwait bool
@ -89,9 +87,6 @@ It will be stored in sidechain when inner ring will accepts it.`,
err = parseAttributes(&cnr, containerAttributes)
commonCmd.ExitOnErr(cmd, "", err)
var basicACL acl.Basic
commonCmd.ExitOnErr(cmd, "decode basic ACL string: %w", basicACL.DecodeString(containerACL))
tok := getSession(cmd)
if tok != nil {
@ -105,7 +100,6 @@ It will be stored in sidechain when inner ring will accepts it.`,
}
cnr.SetPlacementPolicy(*placementPolicy)
cnr.SetBasicACL(basicACL)
var syncContainerPrm internalclient.SyncContainerPrm
syncContainerPrm.SetClient(cli)
@ -163,10 +157,6 @@ func initContainerCreateCmd() {
flags.DurationP(commonflags.Timeout, commonflags.TimeoutShorthand, commonflags.TimeoutDefault, commonflags.TimeoutUsage)
flags.StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, commonflags.WalletPathDefault, commonflags.WalletPathUsage)
flags.StringP(commonflags.Account, commonflags.AccountShorthand, commonflags.AccountDefault, commonflags.AccountUsage)
flags.StringVar(&containerACL, "basic-acl", acl.NamePrivate, fmt.Sprintf("HEX encoded basic ACL value or keywords like '%s', '%s', '%s'",
acl.NamePublicRW, acl.NamePrivate, acl.NamePublicROExtended,
))
flags.StringVarP(&containerPolicy, "policy", "p", "", "QL-encoded or JSON-encoded placement policy or path to file with it")
flags.StringSliceVarP(&containerAttributes, "attributes", "a", nil, "Comma separated pairs of container attributes in form of Key1=Value1,Key2=Value2")
flags.BoolVar(&containerAwait, "await", false, "Block execution until container is persisted")

View file

@ -6,8 +6,11 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/spf13/cobra"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// flags of list command.
@ -51,44 +54,60 @@ var listContainersCmd = &cobra.Command{
var prm internalclient.ListContainersPrm
prm.SetClient(cli)
prm.Account = idUser
res, err := internalclient.ListContainers(cmd.Context(), prm)
commonCmd.ExitOnErr(cmd, "rpc error: %w", err)
prm.OwnerID = idUser
prmGet := internalclient.GetContainerPrm{
Client: cli,
}
var containerIDs []cid.ID
err := internalclient.ListContainersStream(cmd.Context(), prm, func(id cid.ID) bool {
printContainer(cmd, prmGet, id)
return false
})
if err == nil {
return
}
if e, ok := status.FromError(err); ok && e.Code() == codes.Unimplemented {
res, err := internalclient.ListContainers(cmd.Context(), prm)
commonCmd.ExitOnErr(cmd, "rpc error: %w", err)
containerIDs = res.SortedIDList()
} else {
commonCmd.ExitOnErr(cmd, "rpc error: %w", err)
}
containerIDs := res.SortedIDList()
for _, cnrID := range containerIDs {
if flagVarListName == "" && !flagVarListPrintAttr {
cmd.Println(cnrID.String())
continue
}
prmGet.ClientParams.ContainerID = &cnrID
res, err := internalclient.GetContainer(cmd.Context(), prmGet)
if err != nil {
cmd.Printf(" failed to read attributes: %v\n", err)
continue
}
cnr := res.Container()
if cnrName := containerSDK.Name(cnr); flagVarListName != "" && cnrName != flagVarListName {
continue
}
cmd.Println(cnrID.String())
if flagVarListPrintAttr {
cnr.IterateUserAttributes(func(key, val string) {
cmd.Printf(" %s: %s\n", key, val)
})
}
printContainer(cmd, prmGet, cnrID)
}
},
}
func printContainer(cmd *cobra.Command, prmGet internalclient.GetContainerPrm, id cid.ID) {
if flagVarListName == "" && !flagVarListPrintAttr {
cmd.Println(id.String())
return
}
prmGet.ClientParams.ContainerID = &id
res, err := internalclient.GetContainer(cmd.Context(), prmGet)
if err != nil {
cmd.Printf(" failed to read attributes: %v\n", err)
return
}
cnr := res.Container()
if cnrName := containerSDK.Name(cnr); flagVarListName != "" && cnrName != flagVarListName {
return
}
cmd.Println(id.String())
if flagVarListPrintAttr {
cnr.IterateUserAttributes(func(key, val string) {
cmd.Printf(" %s: %s\n", key, val)
})
}
}
func initContainerListContainersCmd() {
commonflags.Init(listContainersCmd)

View file

@ -1,11 +1,10 @@
package container
import (
"bufio"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"
@ -14,6 +13,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"github.com/chzyer/readline"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
@ -163,6 +163,16 @@ func (repl *policyPlaygroundREPL) netMap() netmap.NetMap {
return nm
}
var policyPlaygroundCompleter = readline.NewPrefixCompleter(
readline.PcItem("list"),
readline.PcItem("ls"),
readline.PcItem("add"),
readline.PcItem("load"),
readline.PcItem("remove"),
readline.PcItem("rm"),
readline.PcItem("eval"),
)
func (repl *policyPlaygroundREPL) run() error {
if len(viper.GetString(commonflags.RPC)) > 0 {
key := key.GetOrGenerate(repl.cmd)
@ -189,22 +199,38 @@ func (repl *policyPlaygroundREPL) run() error {
"rm": repl.handleRemove,
"eval": repl.handleEval,
}
for reader := bufio.NewReader(os.Stdin); ; {
fmt.Print("> ")
line, err := reader.ReadString('\n')
rl, err := readline.NewEx(&readline.Config{
Prompt: "> ",
InterruptPrompt: "^C",
AutoComplete: policyPlaygroundCompleter,
})
if err != nil {
return fmt.Errorf("error initializing readline: %w", err)
}
defer rl.Close()
var exit bool
for {
line, err := rl.Readline()
if err != nil {
if err == io.EOF {
return nil
if errors.Is(err, readline.ErrInterrupt) {
if exit {
return nil
}
exit = true
continue
}
return fmt.Errorf("reading line: %v", err)
return fmt.Errorf("reading line: %w", err)
}
exit = false
parts := strings.Fields(line)
if len(parts) == 0 {
continue
}
cmd := parts[0]
handler, exists := cmdHandlers[cmd]
if exists {
if handler, exists := cmdHandlers[cmd]; exists {
if err := handler(parts[1:]); err != nil {
fmt.Printf("error: %v\n", err)
}

View file

@ -1,23 +1,14 @@
package control
import (
"encoding/hex"
"errors"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"github.com/spf13/cobra"
)
const (
ruleFlag = "rule"
pathFlag = "path"
)
var addRuleCmd = &cobra.Command{
Use: "add-rule",
Short: "Add local override",
@ -31,41 +22,12 @@ control add-rule --endpoint ... -w ... --address ... --chain-id ChainID --cid ..
Run: addRule,
}
func parseChain(cmd *cobra.Command) *apechain.Chain {
chainID, _ := cmd.Flags().GetString(chainIDFlag)
hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag)
chainIDRaw := []byte(chainID)
if hexEncoded {
var err error
chainIDRaw, err = hex.DecodeString(chainID)
commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err)
}
chain := new(apechain.Chain)
chain.ID = apechain.ID(chainIDRaw)
if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 {
commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, rules))
} else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" {
commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", util.ParseAPEChainBinaryOrJSON(chain, encPath))
} else {
commonCmd.ExitOnErr(cmd, "parser error", errors.New("rule is not passed"))
}
cmd.Println("Parsed chain:")
util.PrintHumanReadableAPEChain(cmd, chain)
return chain
}
func addRule(cmd *cobra.Command, _ []string) {
pk := key.Get(cmd)
target := parseTarget(cmd)
parsed := parseChain(cmd)
parsed := apeCmd.ParseChain(cmd)
req := &control.AddChainLocalOverrideRequest{
Body: &control.AddChainLocalOverrideRequest_Body{
@ -94,13 +56,13 @@ func initControlAddRuleCmd() {
initControlFlags(addRuleCmd)
ff := addRuleCmd.Flags()
ff.StringArray(ruleFlag, []string{}, "Rule statement")
ff.String(pathFlag, "", "Path to encoded chain in JSON or binary format")
ff.String(chainIDFlag, "", "Assign ID to the parsed chain")
ff.String(targetNameFlag, "", targetNameDesc)
ff.String(targetTypeFlag, "", targetTypeDesc)
_ = addRuleCmd.MarkFlagRequired(targetTypeFlag)
ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex")
ff.StringArray(apeCmd.RuleFlag, []string{}, "Rule statement")
ff.String(apeCmd.PathFlag, "", "Path to encoded chain in JSON or binary format")
ff.String(apeCmd.ChainIDFlag, "", "Assign ID to the parsed chain")
ff.String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc)
ff.String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc)
_ = addRuleCmd.MarkFlagRequired(apeCmd.TargetTypeFlag)
ff.Bool(apeCmd.ChainIDHexFlag, false, "Flag to parse chain ID as hex")
addRuleCmd.MarkFlagsMutuallyExclusive(pathFlag, ruleFlag)
addRuleCmd.MarkFlagsMutuallyExclusive(apeCmd.PathFlag, apeCmd.RuleFlag)
}

View file

@ -1,56 +0,0 @@
package control
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
"github.com/spf13/cobra"
)
const ignoreErrorsFlag = "no-errors"
var evacuateShardCmd = &cobra.Command{
Use: "evacuate",
Short: "Evacuate objects from shard",
Long: "Evacuate objects from shard to other shards",
Run: evacuateShard,
Deprecated: "use frostfs-cli control shards evacuation start",
}
func evacuateShard(cmd *cobra.Command, _ []string) {
pk := key.Get(cmd)
req := &control.EvacuateShardRequest{Body: new(control.EvacuateShardRequest_Body)}
req.Body.Shard_ID = getShardIDList(cmd)
req.Body.IgnoreErrors, _ = cmd.Flags().GetBool(ignoreErrorsFlag)
signRequest(cmd, pk, req)
cli := getClient(cmd, pk)
var resp *control.EvacuateShardResponse
var err error
err = cli.ExecRaw(func(client *client.Client) error {
resp, err = control.EvacuateShard(client, req)
return err
})
commonCmd.ExitOnErr(cmd, "rpc error: %w", err)
cmd.Printf("Objects moved: %d\n", resp.GetBody().GetCount())
verifyResponse(cmd, resp.GetSignature(), resp.GetBody())
cmd.Println("Shard has successfully been evacuated.")
}
func initControlEvacuateShardCmd() {
initControlFlags(evacuateShardCmd)
flags := evacuateShardCmd.Flags()
flags.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding")
flags.Bool(shardAllFlag, false, "Process all shards")
flags.Bool(ignoreErrorsFlag, false, "Skip invalid/unreadable objects")
evacuateShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag)
}

View file

@ -17,10 +17,11 @@ import (
)
const (
awaitFlag = "await"
noProgressFlag = "no-progress"
scopeFlag = "scope"
repOneOnlyFlag = "rep-one-only"
awaitFlag = "await"
noProgressFlag = "no-progress"
scopeFlag = "scope"
repOneOnlyFlag = "rep-one-only"
ignoreErrorsFlag = "no-errors"
containerWorkerCountFlag = "container-worker-count"
objectWorkerCountFlag = "object-worker-count"

View file

@ -4,8 +4,8 @@ import (
"encoding/hex"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apecmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
@ -24,8 +24,8 @@ func getRule(cmd *cobra.Command, _ []string) {
target := parseTarget(cmd)
chainID, _ := cmd.Flags().GetString(chainIDFlag)
hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag)
chainID, _ := cmd.Flags().GetString(apecmd.ChainIDFlag)
hexEncoded, _ := cmd.Flags().GetBool(apecmd.ChainIDHexFlag)
if hexEncoded {
chainIDBytes, err := hex.DecodeString(chainID)
@ -56,16 +56,16 @@ func getRule(cmd *cobra.Command, _ []string) {
var chain apechain.Chain
commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(resp.GetBody().GetChain()))
util.PrintHumanReadableAPEChain(cmd, &chain)
apecmd.PrintHumanReadableAPEChain(cmd, &chain)
}
func initControGetRuleCmd() {
initControlFlags(getRuleCmd)
ff := getRuleCmd.Flags()
ff.String(targetNameFlag, "", targetNameDesc)
ff.String(targetTypeFlag, "", targetTypeDesc)
_ = getRuleCmd.MarkFlagRequired(targetTypeFlag)
ff.String(chainIDFlag, "", "Chain id")
ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex")
ff.String(apecmd.TargetNameFlag, "", apecmd.TargetNameFlagDesc)
ff.String(apecmd.TargetTypeFlag, "", apecmd.TargetTypeFlagDesc)
_ = getRuleCmd.MarkFlagRequired(apecmd.TargetTypeFlag)
ff.String(apecmd.ChainIDFlag, "", "Chain id")
ff.Bool(apecmd.ChainIDHexFlag, false, "Flag to parse chain ID as hex")
}

View file

@ -1,18 +1,16 @@
package control
import (
"errors"
"fmt"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"github.com/nspcc-dev/neo-go/cli/input"
policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
"github.com/spf13/cobra"
)
@ -23,65 +21,25 @@ var listRulesCmd = &cobra.Command{
Run: listRules,
}
const (
defaultNamespace = "root"
namespaceTarget = "namespace"
containerTarget = "container"
userTarget = "user"
groupTarget = "group"
)
const (
targetNameFlag = "target-name"
targetNameDesc = "Resource name in APE resource name format"
targetTypeFlag = "target-type"
targetTypeDesc = "Resource type(container/namespace)"
)
var (
errSettingDefaultValueWasDeclined = errors.New("setting default value was declined")
errUnknownTargetType = errors.New("unknown target type")
)
var engineToControlSvcType = map[policyengine.TargetType]control.ChainTarget_TargetType{
policyengine.Namespace: control.ChainTarget_NAMESPACE,
policyengine.Container: control.ChainTarget_CONTAINER,
policyengine.User: control.ChainTarget_USER,
policyengine.Group: control.ChainTarget_GROUP,
}
func parseTarget(cmd *cobra.Command) *control.ChainTarget {
typ, _ := cmd.Flags().GetString(targetTypeFlag)
name, _ := cmd.Flags().GetString(targetNameFlag)
switch typ {
case namespaceTarget:
if name == "" {
ln, err := input.ReadLine(fmt.Sprintf("Target name is not set. Confirm to use %s namespace (n|Y)> ", defaultNamespace))
commonCmd.ExitOnErr(cmd, "read line error: %w", err)
ln = strings.ToLower(ln)
if len(ln) > 0 && (ln[0] == 'n') {
commonCmd.ExitOnErr(cmd, "read namespace error: %w", errSettingDefaultValueWasDeclined)
}
name = defaultNamespace
}
return &control.ChainTarget{
Name: name,
Type: control.ChainTarget_NAMESPACE,
}
case containerTarget:
var cnr cid.ID
commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(name))
return &control.ChainTarget{
Name: name,
Type: control.ChainTarget_CONTAINER,
}
case userTarget:
return &control.ChainTarget{
Name: name,
Type: control.ChainTarget_USER,
}
case groupTarget:
return &control.ChainTarget{
Name: name,
Type: control.ChainTarget_GROUP,
}
default:
commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType)
target := apeCmd.ParseTarget(cmd)
typ, ok := engineToControlSvcType[target.Type]
if !ok {
commonCmd.ExitOnErr(cmd, "%w", fmt.Errorf("unknown type '%c", target.Type))
}
return &control.ChainTarget{
Name: target.Name,
Type: typ,
}
return nil
}
func listRules(cmd *cobra.Command, _ []string) {
@ -117,7 +75,7 @@ func listRules(cmd *cobra.Command, _ []string) {
for _, c := range chains {
var chain apechain.Chain
commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(c))
util.PrintHumanReadableAPEChain(cmd, &chain)
apeCmd.PrintHumanReadableAPEChain(cmd, &chain)
}
}
@ -125,7 +83,7 @@ func initControlListRulesCmd() {
initControlFlags(listRulesCmd)
ff := listRulesCmd.Flags()
ff.String(targetNameFlag, "", targetNameDesc)
ff.String(targetTypeFlag, "", targetTypeDesc)
_ = listRulesCmd.MarkFlagRequired(targetTypeFlag)
ff.String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc)
ff.String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc)
_ = listRulesCmd.MarkFlagRequired(apeCmd.TargetTypeFlag)
}

View file

@ -2,26 +2,20 @@ package control
import (
"bytes"
"crypto/sha256"
"fmt"
"strconv"
"text/tabwriter"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/spf13/cobra"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
chainNameFlag = "chain-name"
chainNameFlagUsage = "Chain name(ingress|s3)"
)
var listTargetsCmd = &cobra.Command{
Use: "list-targets",
Short: "List local targets",
@ -32,15 +26,11 @@ var listTargetsCmd = &cobra.Command{
func listTargets(cmd *cobra.Command, _ []string) {
pk := key.Get(cmd)
var cnr cid.ID
chainName, _ := cmd.Flags().GetString(chainNameFlag)
rawCID := make([]byte, sha256.Size)
cnr.Encode(rawCID)
chainName := apeCmd.ParseChainName(cmd)
req := &control.ListTargetsLocalOverridesRequest{
Body: &control.ListTargetsLocalOverridesRequest_Body{
ChainName: chainName,
ChainName: string(chainName),
},
}
@ -82,7 +72,7 @@ func initControlListTargetsCmd() {
initControlFlags(listTargetsCmd)
ff := listTargetsCmd.Flags()
ff.String(chainNameFlag, "", chainNameFlagUsage)
ff.String(apeCmd.ChainNameFlag, "", apeCmd.ChainNameFlagDesc)
_ = cobra.MarkFlagRequired(ff, chainNameFlag)
_ = cobra.MarkFlagRequired(ff, apeCmd.ChainNameFlag)
}

View file

@ -6,17 +6,12 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apecmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
"github.com/spf13/cobra"
)
const (
chainIDFlag = "chain-id"
chainIDHexFlag = "chain-id-hex"
allFlag = "all"
)
var (
errEmptyChainID = errors.New("chain id cannot be empty")
@ -30,8 +25,8 @@ var (
func removeRule(cmd *cobra.Command, _ []string) {
pk := key.Get(cmd)
hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag)
removeAll, _ := cmd.Flags().GetBool(allFlag)
hexEncoded, _ := cmd.Flags().GetBool(apecmd.ChainIDHexFlag)
removeAll, _ := cmd.Flags().GetBool(apecmd.AllFlag)
if removeAll {
req := &control.RemoveChainLocalOverridesByTargetRequest{
Body: &control.RemoveChainLocalOverridesByTargetRequest_Body{
@ -52,7 +47,7 @@ func removeRule(cmd *cobra.Command, _ []string) {
return
}
chainID, _ := cmd.Flags().GetString(chainIDFlag)
chainID, _ := cmd.Flags().GetString(apecmd.ChainIDFlag)
if chainID == "" {
commonCmd.ExitOnErr(cmd, "read chain id error: %w", errEmptyChainID)
}
@ -92,11 +87,11 @@ func initControlRemoveRuleCmd() {
initControlFlags(removeRuleCmd)
ff := removeRuleCmd.Flags()
ff.String(targetNameFlag, "", targetNameDesc)
ff.String(targetTypeFlag, "", targetTypeDesc)
_ = removeRuleCmd.MarkFlagRequired(targetTypeFlag)
ff.String(chainIDFlag, "", "Chain id")
ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex")
ff.Bool(allFlag, false, "Remove all chains")
removeRuleCmd.MarkFlagsMutuallyExclusive(allFlag, chainIDFlag)
ff.String(apecmd.TargetNameFlag, "", apecmd.TargetNameFlagDesc)
ff.String(apecmd.TargetTypeFlag, "", apecmd.TargetTypeFlagDesc)
_ = removeRuleCmd.MarkFlagRequired(apecmd.TargetTypeFlag)
ff.String(apecmd.ChainIDFlag, "", apecmd.ChainIDFlagDesc)
ff.Bool(apecmd.ChainIDHexFlag, false, apecmd.ChainIDHexFlagDesc)
ff.Bool(apecmd.AllFlag, false, "Remove all chains")
removeRuleCmd.MarkFlagsMutuallyExclusive(apecmd.AllFlag, apecmd.ChainIDFlag)
}

View file

@ -13,7 +13,6 @@ var shardsCmd = &cobra.Command{
func initControlShardsCmd() {
shardsCmd.AddCommand(listShardsCmd)
shardsCmd.AddCommand(setShardModeCmd)
shardsCmd.AddCommand(evacuateShardCmd)
shardsCmd.AddCommand(evacuationShardCmd)
shardsCmd.AddCommand(flushCacheCmd)
shardsCmd.AddCommand(doctorCmd)
@ -23,7 +22,6 @@ func initControlShardsCmd() {
initControlShardsListCmd()
initControlSetShardModeCmd()
initControlEvacuateShardCmd()
initControlEvacuationShardCmd()
initControlFlushCacheCmd()
initControlDoctorCmd()

View file

@ -9,7 +9,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/spf13/cobra"
@ -43,6 +42,8 @@ func initObjectHashCmd() {
_ = objectHashCmd.MarkFlagRequired(commonflags.OIDFlag)
flags.String("range", "", "Range to take hash from in the form offset1:length1,...")
_ = objectHashCmd.MarkFlagRequired("range")
flags.String("type", hashSha256, "Hash type. Either 'sha256' or 'tz'")
flags.String(getRangeHashSaltFlag, "", "Salt in hex format")
}
@ -66,36 +67,6 @@ func getObjectHash(cmd *cobra.Command, _ []string) {
pk := key.GetOrGenerate(cmd)
cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC)
tz := typ == hashTz
fullHash := len(ranges) == 0
if fullHash {
var headPrm internalclient.HeadObjectPrm
headPrm.SetClient(cli)
Prepare(cmd, &headPrm)
headPrm.SetAddress(objAddr)
// get hash of full payload through HEAD (may be user can do it through dedicated command?)
res, err := internalclient.HeadObject(cmd.Context(), headPrm)
commonCmd.ExitOnErr(cmd, "rpc error: %w", err)
var cs checksum.Checksum
var csSet bool
if tz {
cs, csSet = res.Header().PayloadHomomorphicHash()
} else {
cs, csSet = res.Header().PayloadChecksum()
}
if csSet {
cmd.Println(hex.EncodeToString(cs.Value()))
} else {
cmd.Println("Missing checksum in object header.")
}
return
}
var hashPrm internalclient.HashPayloadRangesPrm
hashPrm.SetClient(cli)
Prepare(cmd, &hashPrm)
@ -104,7 +75,7 @@ func getObjectHash(cmd *cobra.Command, _ []string) {
hashPrm.SetSalt(salt)
hashPrm.SetRanges(ranges)
if tz {
if typ == hashTz {
hashPrm.TZ()
}

View file

@ -1,15 +1,12 @@
package object
import (
"bytes"
"cmp"
"context"
"crypto/ecdsa"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"sync"
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
@ -507,7 +504,6 @@ func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID,
}
func printPlacement(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) {
normilizeObjectNodesResult(objects, result)
if json, _ := cmd.Flags().GetBool(commonflags.JSON); json {
printObjectNodesAsJSON(cmd, objID, objects, result)
} else {
@ -515,34 +511,6 @@ func printPlacement(cmd *cobra.Command, objID oid.ID, objects []phyObject, resul
}
}
func normilizeObjectNodesResult(objects []phyObject, result *objectNodesResult) {
slices.SortFunc(objects, func(lhs, rhs phyObject) int {
if lhs.ecHeader == nil && rhs.ecHeader == nil {
return bytes.Compare(lhs.objectID[:], rhs.objectID[:])
}
if lhs.ecHeader == nil {
return -1
}
if rhs.ecHeader == nil {
return 1
}
if lhs.ecHeader.parent == rhs.ecHeader.parent {
return cmp.Compare(lhs.ecHeader.index, rhs.ecHeader.index)
}
return bytes.Compare(lhs.ecHeader.parent[:], rhs.ecHeader.parent[:])
})
for _, obj := range objects {
op := result.placements[obj.objectID]
slices.SortFunc(op.confirmedNodes, func(lhs, rhs netmapSDK.NodeInfo) int {
return bytes.Compare(lhs.PublicKey(), rhs.PublicKey())
})
slices.SortFunc(op.requiredNodes, func(lhs, rhs netmapSDK.NodeInfo) int {
return bytes.Compare(lhs.PublicKey(), rhs.PublicKey())
})
result.placements[obj.objectID] = op
}
}
func printObjectNodesAsText(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) {
fmt.Fprintf(cmd.OutOrStdout(), "Object %s stores payload in %d data objects:\n", objID.EncodeToString(), len(objects))

View file

@ -6,7 +6,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
apeutil "git.frostfs.info/TrueCloudLab/frostfs-node/internal/ape"
apeutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"github.com/spf13/cobra"
)

View file

@ -1,6 +1,7 @@
package main
import (
"context"
"os"
"os/signal"
"syscall"
@ -46,7 +47,7 @@ func reloadConfig() error {
return logPrm.Reload()
}
func watchForSignal(cancel func()) {
func watchForSignal(ctx context.Context, cancel func()) {
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
@ -58,49 +59,49 @@ func watchForSignal(cancel func()) {
// signals causing application to shut down should have priority over
// reconfiguration signal
case <-ch:
log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)
log.Info(ctx, logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)
cancel()
shutdown()
log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete)
shutdown(ctx)
log.Info(ctx, logs.FrostFSNodeTerminationSignalProcessingIsComplete)
return
case err := <-intErr: // internal application error
log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error()))
log.Info(ctx, logs.FrostFSIRInternalError, zap.String("msg", err.Error()))
cancel()
shutdown()
shutdown(ctx)
return
default:
// block until any signal is receieved
select {
case <-ch:
log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)
log.Info(ctx, logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)
cancel()
shutdown()
log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete)
shutdown(ctx)
log.Info(ctx, logs.FrostFSNodeTerminationSignalProcessingIsComplete)
return
case err := <-intErr: // internal application error
log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error()))
log.Info(ctx, logs.FrostFSIRInternalError, zap.String("msg", err.Error()))
cancel()
shutdown()
shutdown(ctx)
return
case <-sighupCh:
log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration)
if !innerRing.CompareAndSwapHealthStatus(control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) {
log.Info(logs.FrostFSNodeSIGHUPSkip)
log.Info(ctx, logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration)
if !innerRing.CompareAndSwapHealthStatus(ctx, control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) {
log.Info(ctx, logs.FrostFSNodeSIGHUPSkip)
break
}
err := reloadConfig()
if err != nil {
log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err))
log.Error(ctx, logs.FrostFSNodeConfigurationReading, zap.Error(err))
}
pprofCmp.reload()
metricsCmp.reload()
log.Info(logs.FrostFSIRReloadExtraWallets)
pprofCmp.reload(ctx)
metricsCmp.reload(ctx)
log.Info(ctx, logs.FrostFSIRReloadExtraWallets)
err = innerRing.SetExtraWallets(cfg)
if err != nil {
log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err))
log.Error(ctx, logs.FrostFSNodeConfigurationReading, zap.Error(err))
}
innerRing.CompareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY)
log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully)
innerRing.CompareAndSwapHealthStatus(ctx, control.HealthStatus_RECONFIGURING, control.HealthStatus_READY)
log.Info(ctx, logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully)
}
}
}

View file

@ -1,6 +1,7 @@
package main
import (
"context"
"net/http"
"time"
@ -24,8 +25,8 @@ const (
shutdownTimeoutKeyPostfix = ".shutdown_timeout"
)
func (c *httpComponent) init() {
log.Info("init " + c.name)
func (c *httpComponent) init(ctx context.Context) {
log.Info(ctx, "init "+c.name)
c.enabled = cfg.GetBool(c.name + enabledKeyPostfix)
c.address = cfg.GetString(c.name + addressKeyPostfix)
c.shutdownDur = cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix)
@ -39,14 +40,14 @@ func (c *httpComponent) init() {
httputil.WithShutdownTimeout(c.shutdownDur),
)
} else {
log.Info(c.name + " is disabled, skip")
log.Info(ctx, c.name+" is disabled, skip")
c.srv = nil
}
}
func (c *httpComponent) start() {
func (c *httpComponent) start(ctx context.Context) {
if c.srv != nil {
log.Info("start " + c.name)
log.Info(ctx, "start "+c.name)
wg.Add(1)
go func() {
defer wg.Done()
@ -55,10 +56,10 @@ func (c *httpComponent) start() {
}
}
func (c *httpComponent) shutdown() error {
func (c *httpComponent) shutdown(ctx context.Context) error {
if c.srv != nil {
log.Info("shutdown " + c.name)
return c.srv.Shutdown()
log.Info(ctx, "shutdown "+c.name)
return c.srv.Shutdown(ctx)
}
return nil
}
@ -70,17 +71,17 @@ func (c *httpComponent) needReload() bool {
return enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur)
}
func (c *httpComponent) reload() {
log.Info("reload " + c.name)
func (c *httpComponent) reload(ctx context.Context) {
log.Info(ctx, "reload "+c.name)
if c.needReload() {
log.Info(c.name + " config updated")
if err := c.shutdown(); err != nil {
log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()),
log.Info(ctx, c.name+" config updated")
if err := c.shutdown(ctx); err != nil {
log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.Error(err),
)
} else {
c.init()
c.start()
c.init(ctx)
c.start(ctx)
}
}
}

View file

@ -87,48 +87,48 @@ func main() {
ctx, cancel := context.WithCancel(context.Background())
pprofCmp = newPprofComponent()
pprofCmp.init()
pprofCmp.init(ctx)
metricsCmp = newMetricsComponent()
metricsCmp.init()
metricsCmp.init(ctx)
audit.Store(cfg.GetBool("audit.enabled"))
innerRing, err = innerring.New(ctx, log, cfg, intErr, metrics, cmode, audit)
exitErr(err)
pprofCmp.start()
metricsCmp.start()
pprofCmp.start(ctx)
metricsCmp.start(ctx)
// start inner ring
err = innerRing.Start(ctx, intErr)
exitErr(err)
log.Info(logs.CommonApplicationStarted,
log.Info(ctx, logs.CommonApplicationStarted,
zap.String("version", misc.Version))
watchForSignal(cancel)
watchForSignal(ctx, cancel)
<-ctx.Done() // graceful shutdown
log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop)
log.Debug(ctx, logs.FrostFSNodeWaitingForAllProcessesToStop)
wg.Wait()
log.Info(logs.FrostFSIRApplicationStopped)
log.Info(ctx, logs.FrostFSIRApplicationStopped)
}
func shutdown() {
innerRing.Stop()
if err := metricsCmp.shutdown(); err != nil {
log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()),
func shutdown(ctx context.Context) {
innerRing.Stop(ctx)
if err := metricsCmp.shutdown(ctx); err != nil {
log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.Error(err),
)
}
if err := pprofCmp.shutdown(); err != nil {
log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()),
if err := pprofCmp.shutdown(ctx); err != nil {
log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.Error(err),
)
}
if err := sdnotify.ClearStatus(); err != nil {
log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err))
log.Error(ctx, logs.FailedToReportStatusToSystemd, zap.Error(err))
}
}

View file

@ -1,6 +1,7 @@
package main
import (
"context"
"runtime"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
@ -28,8 +29,8 @@ func newPprofComponent() *pprofComponent {
}
}
func (c *pprofComponent) init() {
c.httpComponent.init()
func (c *pprofComponent) init(ctx context.Context) {
c.httpComponent.init(ctx)
if c.enabled {
c.blockRate = cfg.GetInt(pprofBlockRateKey)
@ -51,17 +52,17 @@ func (c *pprofComponent) needReload() bool {
c.enabled && (c.blockRate != blockRate || c.mutexRate != mutexRate)
}
func (c *pprofComponent) reload() {
log.Info("reload " + c.name)
func (c *pprofComponent) reload(ctx context.Context) {
log.Info(ctx, "reload "+c.name)
if c.needReload() {
log.Info(c.name + " config updated")
if err := c.shutdown(); err != nil {
log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()))
log.Info(ctx, c.name+" config updated")
if err := c.shutdown(ctx); err != nil {
log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.Error(err))
return
}
c.init()
c.start()
c.init(ctx)
c.start(ctx)
}
}

View file

@ -28,7 +28,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) {
common.ExitOnErr(cmd, common.Errf("invalid address argument: %w", err))
blz := openBlobovnicza(cmd)
defer blz.Close()
defer blz.Close(cmd.Context())
var prm blobovnicza.GetPrm
prm.SetAddress(addr)

View file

@ -32,7 +32,7 @@ func listFunc(cmd *cobra.Command, _ []string) {
}
blz := openBlobovnicza(cmd)
defer blz.Close()
defer blz.Close(cmd.Context())
err := blobovnicza.IterateAddresses(context.Background(), blz, wAddr)
common.ExitOnErr(cmd, common.Errf("blobovnicza iterator failure: %w", err))

View file

@ -27,7 +27,7 @@ func openBlobovnicza(cmd *cobra.Command) *blobovnicza.Blobovnicza {
blobovnicza.WithPath(vPath),
blobovnicza.WithReadOnly(true),
)
common.ExitOnErr(cmd, blz.Open())
common.ExitOnErr(cmd, blz.Open(cmd.Context()))
return blz
}

View file

@ -31,7 +31,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) {
common.ExitOnErr(cmd, common.Errf("invalid address argument: %w", err))
db := openMeta(cmd)
defer db.Close()
defer db.Close(cmd.Context())
storageID := meta.StorageIDPrm{}
storageID.SetAddress(addr)

View file

@ -19,7 +19,7 @@ func init() {
func listGarbageFunc(cmd *cobra.Command, _ []string) {
db := openMeta(cmd)
defer db.Close()
defer db.Close(cmd.Context())
var garbPrm meta.GarbageIterationPrm
garbPrm.SetHandler(

View file

@ -19,7 +19,7 @@ func init() {
func listGraveyardFunc(cmd *cobra.Command, _ []string) {
db := openMeta(cmd)
defer db.Close()
defer db.Close(cmd.Context())
var gravePrm meta.GraveyardIterationPrm
gravePrm.SetHandler(

View file

@ -3,6 +3,7 @@ package main
import (
"context"
"net"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance"
accountingTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/accounting/grpc"
@ -30,5 +31,27 @@ func initAccountingService(ctx context.Context, c *cfg) {
c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) {
accountingGRPC.RegisterAccountingServiceServer(s, server)
// TODO(@aarifullin): #1487 remove the dual service support.
s.RegisterService(frostFSServiceDesc(accountingGRPC.AccountingService_ServiceDesc), server)
})
}
// frostFSServiceDesc creates a service descriptor with the new namespace for dual service support.
func frostFSServiceDesc(sd grpc.ServiceDesc) *grpc.ServiceDesc {
sdLegacy := new(grpc.ServiceDesc)
*sdLegacy = sd
const (
legacyNamespace = "neo.fs.v2"
apemanagerLegacyNamespace = "frostfs.v2"
newNamespace = "frost.fs"
)
if strings.HasPrefix(sd.ServiceName, legacyNamespace) {
sdLegacy.ServiceName = strings.ReplaceAll(sd.ServiceName, legacyNamespace, newNamespace)
} else if strings.HasPrefix(sd.ServiceName, apemanagerLegacyNamespace) {
sdLegacy.ServiceName = strings.ReplaceAll(sd.ServiceName, apemanagerLegacyNamespace, newNamespace)
}
return sdLegacy
}

View file

@ -26,5 +26,8 @@ func initAPEManagerService(c *cfg) {
c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) {
apemanager_grpc.RegisterAPEManagerServiceServer(s, server)
// TODO(@aarifullin): #1487 remove the dual service support.
s.RegisterService(frostFSServiceDesc(apemanager_grpc.APEManagerService_ServiceDesc), server)
})
}

View file

@ -196,31 +196,6 @@ func (s ttlContainerStorage) DeletionInfo(cnr cid.ID) (*container.DelInfo, error
return s.delInfoCache.get(cnr)
}
type ttlEACLStorage struct {
*ttlNetCache[cid.ID, *container.EACL]
}
func newCachedEACLStorage(v container.EACLSource, ttl time.Duration) ttlEACLStorage {
const eaclCacheSize = 100
lruCnrCache := newNetworkTTLCache(eaclCacheSize, ttl, func(id cid.ID) (*container.EACL, error) {
return v.GetEACL(id)
}, metrics.NewCacheMetrics("eacl"))
return ttlEACLStorage{lruCnrCache}
}
// GetEACL returns eACL value from the cache. If value is missing in the cache
// or expired, then it returns value from side chain and updates cache.
func (s ttlEACLStorage) GetEACL(cnr cid.ID) (*container.EACL, error) {
return s.get(cnr)
}
// InvalidateEACL removes cached eACL value.
func (s ttlEACLStorage) InvalidateEACL(cnr cid.ID) {
s.remove(cnr)
}
type lruNetmapSource struct {
netState netmap.State

View file

@ -397,16 +397,16 @@ type internals struct {
}
// starts node's maintenance.
func (c *cfg) startMaintenance() {
func (c *cfg) startMaintenance(ctx context.Context) {
c.isMaintenance.Store(true)
c.cfgNetmap.state.setControlNetmapStatus(control.NetmapStatus_MAINTENANCE)
c.log.Info(logs.FrostFSNodeStartedLocalNodesMaintenance)
c.log.Info(ctx, logs.FrostFSNodeStartedLocalNodesMaintenance)
}
// stops node's maintenance.
func (c *internals) stopMaintenance() {
func (c *internals) stopMaintenance(ctx context.Context) {
if c.isMaintenance.CompareAndSwap(true, false) {
c.log.Info(logs.FrostFSNodeStoppedLocalNodesMaintenance)
c.log.Info(ctx, logs.FrostFSNodeStoppedLocalNodesMaintenance)
}
}
@ -591,8 +591,6 @@ type cfgMorph struct {
client *client.Client
notaryEnabled bool
// TTL of Sidechain cached values. Non-positive value disables caching.
cacheTTL time.Duration
@ -608,9 +606,10 @@ type cfgAccounting struct {
type cfgContainer struct {
scriptHash neogoutil.Uint160
parsers map[event.Type]event.NotificationParser
subscribers map[event.Type][]event.Handler
workerPool util.WorkerPool // pool for asynchronous handlers
parsers map[event.Type]event.NotificationParser
subscribers map[event.Type][]event.Handler
workerPool util.WorkerPool // pool for asynchronous handlers
containerBatchSize uint32
}
type cfgFrostfsID struct {
@ -642,8 +641,6 @@ type cfgObject struct {
cnrSource container.Source
eaclSource container.EACLSource
cfgAccessPolicyEngine cfgAccessPolicyEngine
pool cfgObjectRoutines
@ -707,7 +704,7 @@ func initCfg(appCfg *config.Config) *cfg {
log, err := logger.NewLogger(logPrm)
fatalOnErr(err)
if loggerconfig.ToLokiConfig(appCfg).Enabled {
log.Logger = log.Logger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
log.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
lokiCore := lokicore.New(core, loggerconfig.ToLokiConfig(appCfg))
return lokiCore
}))
@ -1091,7 +1088,7 @@ func (c *cfg) LocalAddress() network.AddressGroup {
func initLocalStorage(ctx context.Context, c *cfg) {
ls := engine.New(c.engineOpts()...)
addNewEpochAsyncNotificationHandler(c, func(ev event.Event) {
addNewEpochAsyncNotificationHandler(c, func(ctx context.Context, ev event.Event) {
ls.HandleNewEpoch(ctx, ev.(netmap2.NewEpoch).EpochNumber())
})
@ -1105,10 +1102,10 @@ func initLocalStorage(ctx context.Context, c *cfg) {
shard.WithTombstoneSource(c.createTombstoneSource()),
shard.WithContainerInfoProvider(c.createContainerInfoProvider(ctx)))...)
if err != nil {
c.log.Error(logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err))
c.log.Error(ctx, logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err))
} else {
shardsAttached++
c.log.Info(logs.FrostFSNodeShardAttachedToEngine, zap.Stringer("id", id))
c.log.Info(ctx, logs.FrostFSNodeShardAttachedToEngine, zap.Stringer("id", id))
}
}
if shardsAttached == 0 {
@ -1118,23 +1115,23 @@ func initLocalStorage(ctx context.Context, c *cfg) {
c.cfgObject.cfgLocalStorage.localStorage = ls
c.onShutdown(func() {
c.log.Info(logs.FrostFSNodeClosingComponentsOfTheStorageEngine)
c.log.Info(ctx, logs.FrostFSNodeClosingComponentsOfTheStorageEngine)
err := ls.Close(context.WithoutCancel(ctx))
if err != nil {
c.log.Info(logs.FrostFSNodeStorageEngineClosingFailure,
zap.String("error", err.Error()),
c.log.Info(ctx, logs.FrostFSNodeStorageEngineClosingFailure,
zap.Error(err),
)
} else {
c.log.Info(logs.FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully)
c.log.Info(ctx, logs.FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully)
}
})
}
func initAccessPolicyEngine(_ context.Context, c *cfg) {
func initAccessPolicyEngine(ctx context.Context, c *cfg) {
var localOverrideDB chainbase.LocalOverrideDatabase
if nodeconfig.PersistentPolicyRules(c.appCfg).Path() == "" {
c.log.Warn(logs.FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed)
c.log.Warn(ctx, logs.FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed)
localOverrideDB = chainbase.NewInmemoryLocalOverrideDatabase()
} else {
localOverrideDB = chainbase.NewBoltLocalOverrideDatabase(
@ -1159,7 +1156,7 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) {
c.onShutdown(func() {
if err := ape.LocalOverrideDatabaseCore().Close(); err != nil {
c.log.Warn(logs.FrostFSNodeAccessPolicyEngineClosingFailure,
c.log.Warn(ctx, logs.FrostFSNodeAccessPolicyEngineClosingFailure,
zap.Error(err),
)
}
@ -1208,12 +1205,12 @@ func (c *cfg) setContractNodeInfo(ni *netmap.NodeInfo) {
c.cfgNetmap.state.setNodeInfo(ni)
}
func (c *cfg) updateContractNodeInfo(epoch uint64) {
func (c *cfg) updateContractNodeInfo(ctx context.Context, epoch uint64) {
ni, err := c.netmapLocalNodeState(epoch)
if err != nil {
c.log.Error(logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch,
c.log.Error(ctx, logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch,
zap.Uint64("epoch", epoch),
zap.String("error", err.Error()))
zap.Error(err))
return
}
@ -1223,41 +1220,37 @@ func (c *cfg) updateContractNodeInfo(epoch uint64) {
// bootstrapWithState calls "addPeer" method of the Sidechain Netmap contract
// with the binary-encoded information from the current node's configuration.
// The state is set using the provided setter which MUST NOT be nil.
func (c *cfg) bootstrapWithState(stateSetter func(*netmap.NodeInfo)) error {
func (c *cfg) bootstrapWithState(ctx context.Context, state netmap.NodeState) error {
ni := c.cfgNodeInfo.localInfo
stateSetter(&ni)
ni.SetStatus(state)
prm := nmClient.AddPeerPrm{}
prm.SetNodeInfo(ni)
return c.cfgNetmap.wrapper.AddPeer(prm)
return c.cfgNetmap.wrapper.AddPeer(ctx, prm)
}
// bootstrapOnline calls cfg.bootstrapWithState with "online" state.
func bootstrapOnline(c *cfg) error {
return c.bootstrapWithState(func(ni *netmap.NodeInfo) {
ni.SetStatus(netmap.Online)
})
func bootstrapOnline(ctx context.Context, c *cfg) error {
return c.bootstrapWithState(ctx, netmap.Online)
}
// bootstrap calls bootstrapWithState with:
// - "maintenance" state if maintenance is in progress on the current node
// - "online", otherwise
func (c *cfg) bootstrap() error {
func (c *cfg) bootstrap(ctx context.Context) error {
// switch to online except when under maintenance
st := c.cfgNetmap.state.controlNetmapStatus()
if st == control.NetmapStatus_MAINTENANCE {
c.log.Info(logs.FrostFSNodeBootstrappingWithTheMaintenanceState)
return c.bootstrapWithState(func(ni *netmap.NodeInfo) {
ni.SetStatus(netmap.Maintenance)
})
c.log.Info(ctx, logs.FrostFSNodeBootstrappingWithTheMaintenanceState)
return c.bootstrapWithState(ctx, netmap.Maintenance)
}
c.log.Info(logs.FrostFSNodeBootstrappingWithOnlineState,
c.log.Info(ctx, logs.FrostFSNodeBootstrappingWithOnlineState,
zap.Stringer("previous", st),
)
return bootstrapOnline(c)
return bootstrapOnline(ctx, c)
}
// needBootstrap checks if local node should be registered in network on bootup.
@ -1282,19 +1275,19 @@ func (c *cfg) signalWatcher(ctx context.Context) {
// signals causing application to shut down should have priority over
// reconfiguration signal
case <-ch:
c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)
c.log.Info(ctx, logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)
c.shutdown()
c.shutdown(ctx)
c.log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete)
c.log.Info(ctx, logs.FrostFSNodeTerminationSignalProcessingIsComplete)
return
case err := <-c.internalErr: // internal application error
c.log.Warn(logs.FrostFSNodeInternalApplicationError,
c.log.Warn(ctx, logs.FrostFSNodeInternalApplicationError,
zap.String("message", err.Error()))
c.shutdown()
c.shutdown(ctx)
c.log.Info(logs.FrostFSNodeInternalErrorProcessingIsComplete)
c.log.Info(ctx, logs.FrostFSNodeInternalErrorProcessingIsComplete)
return
default:
// block until any signal is receieved
@ -1302,19 +1295,19 @@ func (c *cfg) signalWatcher(ctx context.Context) {
case <-sighupCh:
c.reloadConfig(ctx)
case <-ch:
c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)
c.log.Info(ctx, logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)
c.shutdown()
c.shutdown(ctx)
c.log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete)
c.log.Info(ctx, logs.FrostFSNodeTerminationSignalProcessingIsComplete)
return
case err := <-c.internalErr: // internal application error
c.log.Warn(logs.FrostFSNodeInternalApplicationError,
c.log.Warn(ctx, logs.FrostFSNodeInternalApplicationError,
zap.String("message", err.Error()))
c.shutdown()
c.shutdown(ctx)
c.log.Info(logs.FrostFSNodeInternalErrorProcessingIsComplete)
c.log.Info(ctx, logs.FrostFSNodeInternalErrorProcessingIsComplete)
return
}
}
@ -1322,17 +1315,17 @@ func (c *cfg) signalWatcher(ctx context.Context) {
}
func (c *cfg) reloadConfig(ctx context.Context) {
c.log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration)
c.log.Info(ctx, logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration)
if !c.compareAndSwapHealthStatus(control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) {
c.log.Info(logs.FrostFSNodeSIGHUPSkip)
if !c.compareAndSwapHealthStatus(ctx, control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) {
c.log.Info(ctx, logs.FrostFSNodeSIGHUPSkip)
return
}
defer c.compareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY)
defer c.compareAndSwapHealthStatus(ctx, control.HealthStatus_RECONFIGURING, control.HealthStatus_READY)
err := c.reloadAppConfig()
if err != nil {
c.log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err))
c.log.Error(ctx, logs.FrostFSNodeConfigurationReading, zap.Error(err))
return
}
@ -1343,7 +1336,7 @@ func (c *cfg) reloadConfig(ctx context.Context) {
logPrm, err := c.loggerPrm()
if err != nil {
c.log.Error(logs.FrostFSNodeLoggerConfigurationPreparation, zap.Error(err))
c.log.Error(ctx, logs.FrostFSNodeLoggerConfigurationPreparation, zap.Error(err))
return
}
@ -1364,25 +1357,25 @@ func (c *cfg) reloadConfig(ctx context.Context) {
err = c.cfgObject.cfgLocalStorage.localStorage.Reload(ctx, rcfg)
if err != nil {
c.log.Error(logs.FrostFSNodeStorageEngineConfigurationUpdate, zap.Error(err))
c.log.Error(ctx, logs.FrostFSNodeStorageEngineConfigurationUpdate, zap.Error(err))
return
}
for _, component := range components {
err = component.reloadFunc()
if err != nil {
c.log.Error(logs.FrostFSNodeUpdatedConfigurationApplying,
c.log.Error(ctx, logs.FrostFSNodeUpdatedConfigurationApplying,
zap.String("component", component.name),
zap.Error(err))
}
}
if err := c.dialerSource.Update(internalNetConfig(c.appCfg, c.metricsCollector.MultinetMetrics())); err != nil {
c.log.Error(logs.FailedToUpdateMultinetConfiguration, zap.Error(err))
c.log.Error(ctx, logs.FailedToUpdateMultinetConfiguration, zap.Error(err))
return
}
c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully)
c.log.Info(ctx, logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully)
}
func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp {
@ -1390,7 +1383,7 @@ func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp {
components = append(components, dCmp{"logger", logPrm.Reload})
components = append(components, dCmp{"runtime", func() error {
setRuntimeParameters(c)
setRuntimeParameters(ctx, c)
return nil
}})
components = append(components, dCmp{"audit", func() error {
@ -1405,7 +1398,7 @@ func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp {
}
updated, err := tracing.Setup(ctx, *traceConfig)
if updated {
c.log.Info(logs.FrostFSNodeTracingConfigationUpdated)
c.log.Info(ctx, logs.FrostFSNodeTracingConfigationUpdated)
}
return err
}})
@ -1440,7 +1433,7 @@ func (c *cfg) reloadPools() error {
func (c *cfg) reloadPool(p *ants.Pool, newSize int, name string) {
oldSize := p.Cap()
if oldSize != newSize {
c.log.Info(logs.FrostFSNodePoolConfigurationUpdate, zap.String("field", name),
c.log.Info(context.Background(), logs.FrostFSNodePoolConfigurationUpdate, zap.String("field", name),
zap.Int("old", oldSize), zap.Int("new", newSize))
p.Tune(newSize)
}
@ -1468,7 +1461,7 @@ func (c *cfg) createTombstoneSource() *tombstone.ExpirationChecker {
func (c *cfg) createContainerInfoProvider(ctx context.Context) container.InfoProvider {
return container.NewInfoProvider(func() (container.Source, error) {
c.initMorphComponents(ctx)
cc, err := containerClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0, containerClient.TryNotary())
cc, err := containerClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0)
if err != nil {
return nil, err
}
@ -1476,14 +1469,14 @@ func (c *cfg) createContainerInfoProvider(ctx context.Context) container.InfoPro
})
}
func (c *cfg) shutdown() {
old := c.swapHealthStatus(control.HealthStatus_SHUTTING_DOWN)
func (c *cfg) shutdown(ctx context.Context) {
old := c.swapHealthStatus(ctx, control.HealthStatus_SHUTTING_DOWN)
if old == control.HealthStatus_SHUTTING_DOWN {
c.log.Info(logs.FrostFSNodeShutdownSkip)
c.log.Info(ctx, logs.FrostFSNodeShutdownSkip)
return
}
if old == control.HealthStatus_STARTING {
c.log.Warn(logs.FrostFSNodeShutdownWhenNotReady)
c.log.Warn(ctx, logs.FrostFSNodeShutdownWhenNotReady)
}
c.ctxCancel()
@ -1493,6 +1486,6 @@ func (c *cfg) shutdown() {
}
if err := sdnotify.ClearStatus(); err != nil {
c.log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err))
c.log.Error(ctx, logs.FailedToReportStatusToSystemd, zap.Error(err))
}
}

View file

@ -0,0 +1,27 @@
package containerconfig
import "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
const (
subsection = "container"
listStreamSubsection = "list_stream"
// ContainerBatchSizeDefault represents he maximum amount of containers to send via stream at once.
ContainerBatchSizeDefault = 1000
)
// ContainerBatchSize returns the value of "batch_size" config parameter
// from "list_stream" subsection of "container" section.
//
// Returns ContainerBatchSizeDefault if the value is missing or if
// the value is not positive integer.
func ContainerBatchSize(c *config.Config) uint32 {
if c.Sub(subsection).Sub(listStreamSubsection).Value("batch_size") == nil {
return ContainerBatchSizeDefault
}
size := config.Uint32Safe(c.Sub(subsection).Sub(listStreamSubsection), "batch_size")
if size == 0 {
return ContainerBatchSizeDefault
}
return size
}

View file

@ -0,0 +1,27 @@
package containerconfig_test
import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
containerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/container"
configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test"
"github.com/stretchr/testify/require"
)
func TestContainerSection(t *testing.T) {
t.Run("defaults", func(t *testing.T) {
empty := configtest.EmptyConfig()
require.Equal(t, uint32(containerconfig.ContainerBatchSizeDefault), containerconfig.ContainerBatchSize(empty))
})
const path = "../../../../config/example/node"
fileConfigTest := func(c *config.Config) {
require.Equal(t, uint32(500), containerconfig.ContainerBatchSize(c))
}
configtest.ForEachFileType(path, fileConfigTest)
t.Run("ENV", func(t *testing.T) {
configtest.ForEnvFileType(t, path, fileConfigTest)
})
}

View file

@ -41,6 +41,10 @@ func IterateShards(c *config.Config, required bool, f func(*shardconfig.Config)
c.Sub(si),
)
if sc.Mode() == mode.Disabled {
continue
}
// Path for the blobstor can't be present in the default section, because different shards
// must have different paths, so if it is missing, the shard is not here.
// At the same time checking for "blobstor" section doesn't work proper
@ -50,10 +54,6 @@ func IterateShards(c *config.Config, required bool, f func(*shardconfig.Config)
}
(*config.Config)(sc).SetDefault(def)
if sc.Mode() == mode.Disabled {
continue
}
if err := f(sc); err != nil {
return err
}

View file

@ -18,6 +18,22 @@ import (
"github.com/stretchr/testify/require"
)
func TestIterateShards(t *testing.T) {
fileConfigTest := func(c *config.Config) {
var res []string
require.NoError(t,
engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error {
res = append(res, sc.Metabase().Path())
return nil
}))
require.Equal(t, []string{"abc", "xyz"}, res)
}
const cfgDir = "./testdata/shards"
configtest.ForEachFileType(cfgDir, fileConfigTest)
configtest.ForEnvFileType(t, cfgDir, fileConfigTest)
}
func TestEngineSection(t *testing.T) {
t.Run("defaults", func(t *testing.T) {
empty := configtest.EmptyConfig()

View file

@ -0,0 +1,3 @@
FROSTFS_STORAGE_SHARD_0_METABASE_PATH=abc
FROSTFS_STORAGE_SHARD_1_MODE=disabled
FROSTFS_STORAGE_SHARD_2_METABASE_PATH=xyz

View file

@ -0,0 +1,13 @@
{
"storage.shard": {
"0": {
"metabase.path": "abc"
},
"1": {
"mode": "disabled"
},
"2": {
"metabase.path": "xyz"
}
}
}

View file

@ -0,0 +1,7 @@
storage.shard:
0:
metabase.path: abc
1:
mode: disabled
2:
metabase.path: xyz

View file

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"os"
"strconv"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
"git.frostfs.info/TrueCloudLab/frostfs-node/misc"
@ -24,6 +25,7 @@ func ToTracingConfig(c *config.Config) (*tracing.Config, error) {
Service: "frostfs-node",
InstanceID: getInstanceIDOrDefault(c),
Version: misc.Version,
Attributes: make(map[string]string),
}
if trustedCa := config.StringSafe(c.Sub(subsection), "trusted_ca"); trustedCa != "" {
@ -38,11 +40,30 @@ func ToTracingConfig(c *config.Config) (*tracing.Config, error) {
}
conf.ServerCaCertPool = certPool
}
i := uint64(0)
for ; ; i++ {
si := strconv.FormatUint(i, 10)
ac := c.Sub(subsection).Sub("attributes").Sub(si)
k := config.StringSafe(ac, "key")
if k == "" {
break
}
v := config.StringSafe(ac, "value")
if v == "" {
return nil, fmt.Errorf("empty tracing attribute value for key %s", k)
}
if _, ok := conf.Attributes[k]; ok {
return nil, fmt.Errorf("tracing attribute key %s defined more than once", k)
}
conf.Attributes[k] = v
}
return conf, nil
}
func getInstanceIDOrDefault(c *config.Config) string {
s := config.StringSlice(c.Sub("node"), "addresses")
s := config.StringSliceSafe(c.Sub("node"), "addresses")
if len(s) > 0 {
return s[0]
}

View file

@ -0,0 +1,46 @@
package tracing
import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
"github.com/stretchr/testify/require"
)
func TestTracingSection(t *testing.T) {
t.Run("defaults", func(t *testing.T) {
tc, err := ToTracingConfig(configtest.EmptyConfig())
require.NoError(t, err)
require.Equal(t, false, tc.Enabled)
require.Equal(t, tracing.Exporter(""), tc.Exporter)
require.Equal(t, "", tc.Endpoint)
require.Equal(t, "frostfs-node", tc.Service)
require.Equal(t, "", tc.InstanceID)
require.Nil(t, tc.ServerCaCertPool)
require.Empty(t, tc.Attributes)
})
const path = "../../../../config/example/node"
fileConfigTest := func(c *config.Config) {
tc, err := ToTracingConfig(c)
require.NoError(t, err)
require.Equal(t, true, tc.Enabled)
require.Equal(t, tracing.OTLPgRPCExporter, tc.Exporter)
require.Equal(t, "localhost", tc.Endpoint)
require.Equal(t, "frostfs-node", tc.Service)
require.Nil(t, tc.ServerCaCertPool)
require.EqualValues(t, map[string]string{
"key0": "value",
"key1": "value",
}, tc.Attributes)
}
configtest.ForEachFileType(path, fileConfigTest)
t.Run("ENV", func(t *testing.T) {
configtest.ForEnvFileType(t, path, fileConfigTest)
})
}

View file

@ -10,6 +10,8 @@ import (
const (
subsection = "tree"
SyncBatchSizeDefault = 1000
)
// TreeConfig is a wrapper over "tree" config section
@ -74,6 +76,17 @@ func (c TreeConfig) SyncInterval() time.Duration {
return config.DurationSafe(c.cfg, "sync_interval")
}
// SyncBatchSize returns the value of "sync_batch_size"
// config parameter from the "tree" section.
//
// Returns `SyncBatchSizeDefault` if config value is not specified.
func (c TreeConfig) SyncBatchSize() int {
if v := config.IntSafe(c.cfg, "sync_batch_size"); v > 0 {
return int(v)
}
return SyncBatchSizeDefault
}
// AuthorizedKeys parses and returns an array of "authorized_keys" config
// parameter from "tree" section.
//

View file

@ -44,6 +44,7 @@ func TestTreeSection(t *testing.T) {
require.Equal(t, 32, treeSec.ReplicationWorkerCount())
require.Equal(t, 5*time.Second, treeSec.ReplicationTimeout())
require.Equal(t, time.Hour, treeSec.SyncInterval())
require.Equal(t, 2000, treeSec.SyncBatchSize())
require.Equal(t, expectedKeys, treeSec.AuthorizedKeys())
}

View file

@ -5,6 +5,7 @@ import (
"context"
"net"
containerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/container"
morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics"
@ -28,7 +29,7 @@ import (
func initContainerService(_ context.Context, c *cfg) {
// container wrapper that tries to invoke notary
// requests if chain is configured so
wrap, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0, cntClient.TryNotary())
wrap, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0)
fatalOnErr(err)
c.shared.cnrClient = wrap
@ -47,6 +48,7 @@ func initContainerService(_ context.Context, c *cfg) {
}
c.shared.frostfsidClient = frostfsIDSubjectProvider
c.cfgContainer.containerBatchSize = containerconfig.ContainerBatchSize(c.appCfg)
defaultChainRouter := engine.NewDefaultChainRouterWithLocalOverrides(
c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.MorphRuleChainStorage(),
@ -56,7 +58,9 @@ func initContainerService(_ context.Context, c *cfg) {
&c.key.PrivateKey,
containerService.NewAPEServer(defaultChainRouter, cnrRdr,
newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, c.shared.frostfsidClient,
containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc),
containerService.NewSplitterService(
c.cfgContainer.containerBatchSize, c.respSvc,
containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc)),
),
)
service = containerService.NewAuditService(service, c.log, c.audit)
@ -64,16 +68,15 @@ func initContainerService(_ context.Context, c *cfg) {
c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) {
containerGRPC.RegisterContainerServiceServer(s, server)
// TODO(@aarifullin): #1487 remove the dual service support.
s.RegisterService(frostFSServiceDesc(containerGRPC.ContainerService_ServiceDesc), server)
})
c.cfgObject.cfgLocalStorage.localStorage.SetContainerSource(cnrRdr)
}
func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc containerCore.Source) (*morphContainerReader, *morphContainerWriter) {
eACLFetcher := &morphEACLFetcher{
w: client,
}
cnrRdr := new(morphContainerReader)
cnrWrt := &morphContainerWriter{
@ -81,8 +84,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c
}
if c.cfgMorph.cacheTTL <= 0 {
c.cfgObject.eaclSource = eACLFetcher
cnrRdr.eacl = eACLFetcher
c.cfgObject.cnrSource = cnrSrc
cnrRdr.src = cnrSrc
cnrRdr.lister = client
@ -92,7 +93,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c
if c.cfgMorph.containerCacheSize > 0 {
containerCache := newCachedContainerStorage(cnrSrc, c.cfgMorph.cacheTTL, c.cfgMorph.containerCacheSize)
subscribeToContainerCreation(c, func(e event.Event) {
subscribeToContainerCreation(c, func(ctx context.Context, e event.Event) {
ev := e.(containerEvent.PutSuccess)
// read owner of the created container in order to update the reading cache.
@ -105,32 +106,28 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c
} else {
// unlike removal, we expect successful receive of the container
// after successful creation, so logging can be useful
c.log.Error(logs.FrostFSNodeReadNewlyCreatedContainerAfterTheNotification,
c.log.Error(ctx, logs.FrostFSNodeReadNewlyCreatedContainerAfterTheNotification,
zap.Stringer("id", ev.ID),
zap.Error(err),
)
}
c.log.Debug(logs.FrostFSNodeContainerCreationEventsReceipt,
c.log.Debug(ctx, logs.FrostFSNodeContainerCreationEventsReceipt,
zap.Stringer("id", ev.ID),
)
})
subscribeToContainerRemoval(c, func(e event.Event) {
subscribeToContainerRemoval(c, func(ctx context.Context, e event.Event) {
ev := e.(containerEvent.DeleteSuccess)
containerCache.handleRemoval(ev.ID)
c.log.Debug(logs.FrostFSNodeContainerRemovalEventsReceipt,
c.log.Debug(ctx, logs.FrostFSNodeContainerRemovalEventsReceipt,
zap.Stringer("id", ev.ID),
)
})
c.cfgObject.cnrSource = containerCache
}
cachedEACLStorage := newCachedEACLStorage(eACLFetcher, c.cfgMorph.cacheTTL)
c.cfgObject.eaclSource = cachedEACLStorage
cnrRdr.lister = client
cnrRdr.eacl = c.cfgObject.eaclSource
cnrRdr.src = c.cfgObject.cnrSource
}
@ -221,8 +218,6 @@ func (c *cfg) ExternalAddresses() []string {
// implements interface required by container service provided by morph executor.
type morphContainerReader struct {
eacl containerCore.EACLSource
src containerCore.Source
lister interface {
@ -238,10 +233,6 @@ func (x *morphContainerReader) DeletionInfo(id cid.ID) (*containerCore.DelInfo,
return x.src.DeletionInfo(id)
}
func (x *morphContainerReader) GetEACL(id cid.ID) (*containerCore.EACL, error) {
return x.eacl.GetEACL(id)
}
func (x *morphContainerReader) ContainersOf(id *user.ID) ([]cid.ID, error) {
return x.lister.ContainersOf(id)
}
@ -250,10 +241,10 @@ type morphContainerWriter struct {
neoClient *cntClient.Client
}
func (m morphContainerWriter) Put(cnr containerCore.Container) (*cid.ID, error) {
return cntClient.Put(m.neoClient, cnr)
func (m morphContainerWriter) Put(ctx context.Context, cnr containerCore.Container) (*cid.ID, error) {
return cntClient.Put(ctx, m.neoClient, cnr)
}
func (m morphContainerWriter) Delete(witness containerCore.RemovalWitness) error {
return cntClient.Delete(m.neoClient, witness)
func (m morphContainerWriter) Delete(ctx context.Context, witness containerCore.RemovalWitness) error {
return cntClient.Delete(ctx, m.neoClient, witness)
}

View file

@ -16,7 +16,7 @@ import (
const serviceNameControl = "control"
func initControlService(c *cfg) {
func initControlService(ctx context.Context, c *cfg) {
endpoint := controlconfig.GRPC(c.appCfg).Endpoint()
if endpoint == controlconfig.GRPCEndpointDefault {
return
@ -46,21 +46,21 @@ func initControlService(c *cfg) {
lis, err := net.Listen("tcp", endpoint)
if err != nil {
c.log.Error(logs.FrostFSNodeCantListenGRPCEndpointControl, zap.Error(err))
c.log.Error(ctx, logs.FrostFSNodeCantListenGRPCEndpointControl, zap.Error(err))
return
}
c.cfgControlService.server = grpc.NewServer()
c.onShutdown(func() {
stopGRPC("FrostFS Control API", c.cfgControlService.server, c.log)
stopGRPC(ctx, "FrostFS Control API", c.cfgControlService.server, c.log)
})
control.RegisterControlServiceServer(c.cfgControlService.server, ctlSvc)
c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) {
runAndLog(ctx, c, serviceNameControl, false, func(context.Context, *cfg) {
c.log.Info(logs.FrostFSNodeStartListeningEndpoint,
c.log.Info(ctx, logs.FrostFSNodeStartListeningEndpoint,
zap.String("service", serviceNameControl),
zap.String("endpoint", endpoint))
fatalOnErr(c.cfgControlService.server.Serve(lis))
@ -72,23 +72,23 @@ func (c *cfg) NetmapStatus() control.NetmapStatus {
return c.cfgNetmap.state.controlNetmapStatus()
}
func (c *cfg) setHealthStatus(st control.HealthStatus) {
c.notifySystemd(st)
func (c *cfg) setHealthStatus(ctx context.Context, st control.HealthStatus) {
c.notifySystemd(ctx, st)
c.healthStatus.Store(int32(st))
c.metricsCollector.State().SetHealth(int32(st))
}
func (c *cfg) compareAndSwapHealthStatus(oldSt, newSt control.HealthStatus) (swapped bool) {
func (c *cfg) compareAndSwapHealthStatus(ctx context.Context, oldSt, newSt control.HealthStatus) (swapped bool) {
if swapped = c.healthStatus.CompareAndSwap(int32(oldSt), int32(newSt)); swapped {
c.notifySystemd(newSt)
c.notifySystemd(ctx, newSt)
c.metricsCollector.State().SetHealth(int32(newSt))
}
return
}
func (c *cfg) swapHealthStatus(st control.HealthStatus) (old control.HealthStatus) {
func (c *cfg) swapHealthStatus(ctx context.Context, st control.HealthStatus) (old control.HealthStatus) {
old = control.HealthStatus(c.healthStatus.Swap(int32(st)))
c.notifySystemd(st)
c.notifySystemd(ctx, st)
c.metricsCollector.State().SetHealth(int32(st))
return
}
@ -97,7 +97,7 @@ func (c *cfg) HealthStatus() control.HealthStatus {
return control.HealthStatus(c.healthStatus.Load())
}
func (c *cfg) notifySystemd(st control.HealthStatus) {
func (c *cfg) notifySystemd(ctx context.Context, st control.HealthStatus) {
if !c.sdNotify {
return
}
@ -113,6 +113,6 @@ func (c *cfg) notifySystemd(st control.HealthStatus) {
err = sdnotify.Status(fmt.Sprintf("%v", st))
}
if err != nil {
c.log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err))
c.log.Error(ctx, logs.FailedToReportStatusToSystemd, zap.Error(err))
}
}

View file

@ -1,6 +1,7 @@
package main
import (
"context"
"crypto/tls"
"errors"
"net"
@ -18,11 +19,11 @@ import (
const maxRecvMsgSize = 256 << 20
func initGRPC(c *cfg) {
func initGRPC(ctx context.Context, c *cfg) {
var endpointsToReconnect []string
var successCount int
grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) {
serverOpts, ok := getGrpcServerOpts(c, sc)
serverOpts, ok := getGrpcServerOpts(ctx, c, sc)
if !ok {
return
}
@ -30,7 +31,7 @@ func initGRPC(c *cfg) {
lis, err := net.Listen("tcp", sc.Endpoint())
if err != nil {
c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(sc.Endpoint())
c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err))
c.log.Error(ctx, logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err))
endpointsToReconnect = append(endpointsToReconnect, sc.Endpoint())
return
}
@ -39,7 +40,7 @@ func initGRPC(c *cfg) {
srv := grpc.NewServer(serverOpts...)
c.onShutdown(func() {
stopGRPC("FrostFS Public API", srv, c.log)
stopGRPC(ctx, "FrostFS Public API", srv, c.log)
})
c.cfgGRPC.append(sc.Endpoint(), lis, srv)
@ -52,11 +53,11 @@ func initGRPC(c *cfg) {
c.cfgGRPC.reconnectTimeout = grpcconfig.ReconnectTimeout(c.appCfg)
for _, endpoint := range endpointsToReconnect {
scheduleReconnect(endpoint, c)
scheduleReconnect(ctx, endpoint, c)
}
}
func scheduleReconnect(endpoint string, c *cfg) {
func scheduleReconnect(ctx context.Context, endpoint string, c *cfg) {
c.wg.Add(1)
go func() {
defer c.wg.Done()
@ -65,7 +66,7 @@ func scheduleReconnect(endpoint string, c *cfg) {
for {
select {
case <-t.C:
if tryReconnect(endpoint, c) {
if tryReconnect(ctx, endpoint, c) {
return
}
case <-c.done:
@ -75,20 +76,20 @@ func scheduleReconnect(endpoint string, c *cfg) {
}()
}
func tryReconnect(endpoint string, c *cfg) bool {
c.log.Info(logs.FrostFSNodeGRPCReconnecting, zap.String("endpoint", endpoint))
func tryReconnect(ctx context.Context, endpoint string, c *cfg) bool {
c.log.Info(ctx, logs.FrostFSNodeGRPCReconnecting, zap.String("endpoint", endpoint))
serverOpts, found := getGRPCEndpointOpts(endpoint, c)
serverOpts, found := getGRPCEndpointOpts(ctx, endpoint, c)
if !found {
c.log.Warn(logs.FrostFSNodeGRPCServerConfigNotFound, zap.String("endpoint", endpoint))
c.log.Warn(ctx, logs.FrostFSNodeGRPCServerConfigNotFound, zap.String("endpoint", endpoint))
return true
}
lis, err := net.Listen("tcp", endpoint)
if err != nil {
c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(endpoint)
c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err))
c.log.Warn(logs.FrostFSNodeGRPCReconnectFailed, zap.Duration("next_try_in", c.cfgGRPC.reconnectTimeout))
c.log.Error(ctx, logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err))
c.log.Warn(ctx, logs.FrostFSNodeGRPCReconnectFailed, zap.Duration("next_try_in", c.cfgGRPC.reconnectTimeout))
return false
}
c.metricsCollector.GrpcServerMetrics().MarkHealthy(endpoint)
@ -96,16 +97,16 @@ func tryReconnect(endpoint string, c *cfg) bool {
srv := grpc.NewServer(serverOpts...)
c.onShutdown(func() {
stopGRPC("FrostFS Public API", srv, c.log)
stopGRPC(ctx, "FrostFS Public API", srv, c.log)
})
c.cfgGRPC.appendAndHandle(endpoint, lis, srv)
c.log.Info(logs.FrostFSNodeGRPCReconnectedSuccessfully, zap.String("endpoint", endpoint))
c.log.Info(ctx, logs.FrostFSNodeGRPCReconnectedSuccessfully, zap.String("endpoint", endpoint))
return true
}
func getGRPCEndpointOpts(endpoint string, c *cfg) (result []grpc.ServerOption, found bool) {
func getGRPCEndpointOpts(ctx context.Context, endpoint string, c *cfg) (result []grpc.ServerOption, found bool) {
unlock := c.LockAppConfigShared()
defer unlock()
grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) {
@ -116,7 +117,7 @@ func getGRPCEndpointOpts(endpoint string, c *cfg) (result []grpc.ServerOption, f
return
}
var ok bool
result, ok = getGrpcServerOpts(c, sc)
result, ok = getGrpcServerOpts(ctx, c, sc)
if !ok {
return
}
@ -125,7 +126,7 @@ func getGRPCEndpointOpts(endpoint string, c *cfg) (result []grpc.ServerOption, f
return
}
func getGrpcServerOpts(c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool) {
func getGrpcServerOpts(ctx context.Context, c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool) {
serverOpts := []grpc.ServerOption{
grpc.MaxRecvMsgSize(maxRecvMsgSize),
grpc.ChainUnaryInterceptor(
@ -143,7 +144,7 @@ func getGrpcServerOpts(c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool
if tlsCfg != nil {
cert, err := tls.LoadX509KeyPair(tlsCfg.CertificateFile(), tlsCfg.KeyFile())
if err != nil {
c.log.Error(logs.FrostFSNodeCouldNotReadCertificateFromFile, zap.Error(err))
c.log.Error(ctx, logs.FrostFSNodeCouldNotReadCertificateFromFile, zap.Error(err))
return nil, false
}
@ -174,38 +175,38 @@ func getGrpcServerOpts(c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool
return serverOpts, true
}
func serveGRPC(c *cfg) {
func serveGRPC(ctx context.Context, c *cfg) {
c.cfgGRPC.performAndSave(func(e string, l net.Listener, s *grpc.Server) {
c.wg.Add(1)
go func() {
defer func() {
c.log.Info(logs.FrostFSNodeStopListeningGRPCEndpoint,
c.log.Info(ctx, logs.FrostFSNodeStopListeningGRPCEndpoint,
zap.Stringer("endpoint", l.Addr()),
)
c.wg.Done()
}()
c.log.Info(logs.FrostFSNodeStartListeningEndpoint,
c.log.Info(ctx, logs.FrostFSNodeStartListeningEndpoint,
zap.String("service", "gRPC"),
zap.Stringer("endpoint", l.Addr()),
)
if err := s.Serve(l); err != nil {
c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(e)
c.log.Error(logs.FrostFSNodeGRPCServerError, zap.Error(err))
c.log.Error(ctx, logs.FrostFSNodeGRPCServerError, zap.Error(err))
c.cfgGRPC.dropConnection(e)
scheduleReconnect(e, c)
scheduleReconnect(ctx, e, c)
}
}()
})
}
func stopGRPC(name string, s *grpc.Server, l *logger.Logger) {
l = &logger.Logger{Logger: l.With(zap.String("name", name))}
func stopGRPC(ctx context.Context, name string, s *grpc.Server, l *logger.Logger) {
l = l.With(zap.String("name", name))
l.Info(logs.FrostFSNodeStoppingGRPCServer)
l.Info(ctx, logs.FrostFSNodeStoppingGRPCServer)
// GracefulStop() may freeze forever, see #1270
done := make(chan struct{})
@ -217,9 +218,9 @@ func stopGRPC(name string, s *grpc.Server, l *logger.Logger) {
select {
case <-done:
case <-time.After(1 * time.Minute):
l.Info(logs.FrostFSNodeGRPCCannotShutdownGracefullyForcingStop)
l.Info(ctx, logs.FrostFSNodeGRPCCannotShutdownGracefullyForcingStop)
s.Stop()
}
l.Info(logs.FrostFSNodeGRPCServerStoppedSuccessfully)
l.Info(ctx, logs.FrostFSNodeGRPCServerStoppedSuccessfully)
}

View file

@ -20,9 +20,9 @@ type httpComponent struct {
preReload func(c *cfg)
}
func (cmp *httpComponent) init(c *cfg) {
func (cmp *httpComponent) init(ctx context.Context, c *cfg) {
if !cmp.enabled {
c.log.Info(cmp.name + " is disabled")
c.log.Info(ctx, cmp.name+" is disabled")
return
}
// Init server with parameters
@ -39,14 +39,14 @@ func (cmp *httpComponent) init(c *cfg) {
go func() {
defer c.wg.Done()
c.log.Info(logs.FrostFSNodeStartListeningEndpoint,
c.log.Info(ctx, logs.FrostFSNodeStartListeningEndpoint,
zap.String("service", cmp.name),
zap.String("endpoint", cmp.address))
fatalOnErr(srv.Serve())
}()
c.closers = append(c.closers, closer{
cmp.name,
func() { stopAndLog(c, cmp.name, srv.Shutdown) },
func() { stopAndLog(ctx, c, cmp.name, srv.Shutdown) },
})
}
@ -62,7 +62,7 @@ func (cmp *httpComponent) reload(ctx context.Context) error {
// Cleanup
delCloser(cmp.cfg, cmp.name)
// Init server with new parameters
cmp.init(cmp.cfg)
cmp.init(ctx, cmp.cfg)
// Start worker
if cmp.enabled {
startWorker(ctx, cmp.cfg, *getWorker(cmp.cfg, cmp.name))

View file

@ -61,21 +61,21 @@ func main() {
var ctx context.Context
ctx, c.ctxCancel = context.WithCancel(context.Background())
c.setHealthStatus(control.HealthStatus_STARTING)
c.setHealthStatus(ctx, control.HealthStatus_STARTING)
initApp(ctx, c)
bootUp(ctx, c)
c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
c.compareAndSwapHealthStatus(ctx, control.HealthStatus_STARTING, control.HealthStatus_READY)
wait(c)
}
func initAndLog(c *cfg, name string, initializer func(*cfg)) {
c.log.Info(fmt.Sprintf("initializing %s service...", name))
func initAndLog(ctx context.Context, c *cfg, name string, initializer func(*cfg)) {
c.log.Info(ctx, fmt.Sprintf("initializing %s service...", name))
initializer(c)
c.log.Info(name + " service has been successfully initialized")
c.log.Info(ctx, name+" service has been successfully initialized")
}
func initApp(ctx context.Context, c *cfg) {
@ -85,72 +85,72 @@ func initApp(ctx context.Context, c *cfg) {
c.wg.Done()
}()
setRuntimeParameters(c)
setRuntimeParameters(ctx, c)
metrics, _ := metricsComponent(c)
initAndLog(c, "profiler", initProfilerService)
initAndLog(c, metrics.name, metrics.init)
initAndLog(ctx, c, "profiler", func(c *cfg) { initProfilerService(ctx, c) })
initAndLog(ctx, c, metrics.name, func(c *cfg) { metrics.init(ctx, c) })
initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) })
initAndLog(ctx, c, "tracing", func(c *cfg) { initTracing(ctx, c) })
initLocalStorage(ctx, c)
initAndLog(c, "storage engine", func(c *cfg) {
initAndLog(ctx, c, "storage engine", func(c *cfg) {
fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Open(ctx))
fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Init(ctx))
})
initAndLog(c, "gRPC", initGRPC)
initAndLog(c, "netmap", func(c *cfg) { initNetmapService(ctx, c) })
initAndLog(ctx, c, "gRPC", func(c *cfg) { initGRPC(ctx, c) })
initAndLog(ctx, c, "netmap", func(c *cfg) { initNetmapService(ctx, c) })
initAccessPolicyEngine(ctx, c)
initAndLog(c, "access policy engine", func(c *cfg) {
initAndLog(ctx, c, "access policy engine", func(c *cfg) {
fatalOnErr(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalOverrideDatabaseCore().Open(ctx))
fatalOnErr(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalOverrideDatabaseCore().Init())
})
initAndLog(c, "accounting", func(c *cfg) { initAccountingService(ctx, c) })
initAndLog(c, "container", func(c *cfg) { initContainerService(ctx, c) })
initAndLog(c, "session", initSessionService)
initAndLog(c, "object", initObjectService)
initAndLog(c, "tree", initTreeService)
initAndLog(c, "apemanager", initAPEManagerService)
initAndLog(c, "control", initControlService)
initAndLog(ctx, c, "accounting", func(c *cfg) { initAccountingService(ctx, c) })
initAndLog(ctx, c, "container", func(c *cfg) { initContainerService(ctx, c) })
initAndLog(ctx, c, "session", initSessionService)
initAndLog(ctx, c, "object", initObjectService)
initAndLog(ctx, c, "tree", initTreeService)
initAndLog(ctx, c, "apemanager", initAPEManagerService)
initAndLog(ctx, c, "control", func(c *cfg) { initControlService(ctx, c) })
initAndLog(c, "morph notifications", func(c *cfg) { listenMorphNotifications(ctx, c) })
initAndLog(ctx, c, "morph notifications", func(c *cfg) { listenMorphNotifications(ctx, c) })
}
func runAndLog(ctx context.Context, c *cfg, name string, logSuccess bool, starter func(context.Context, *cfg)) {
c.log.Info(fmt.Sprintf("starting %s service...", name))
c.log.Info(ctx, fmt.Sprintf("starting %s service...", name))
starter(ctx, c)
if logSuccess {
c.log.Info(name + " service started successfully")
c.log.Info(ctx, name+" service started successfully")
}
}
func stopAndLog(c *cfg, name string, stopper func() error) {
c.log.Debug(fmt.Sprintf("shutting down %s service", name))
func stopAndLog(ctx context.Context, c *cfg, name string, stopper func(context.Context) error) {
c.log.Debug(ctx, fmt.Sprintf("shutting down %s service", name))
err := stopper()
err := stopper(ctx)
if err != nil {
c.log.Debug(fmt.Sprintf("could not shutdown %s server", name),
zap.String("error", err.Error()),
c.log.Debug(ctx, fmt.Sprintf("could not shutdown %s server", name),
zap.Error(err),
)
}
c.log.Debug(name + " service has been stopped")
c.log.Debug(ctx, name+" service has been stopped")
}
func bootUp(ctx context.Context, c *cfg) {
runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) })
runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(ctx, c) })
runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit)
bootstrapNode(c)
bootstrapNode(ctx, c)
startWorkers(ctx, c)
}
func wait(c *cfg) {
c.log.Info(logs.CommonApplicationStarted,
c.log.Info(context.Background(), logs.CommonApplicationStarted,
zap.String("version", misc.Version))
<-c.done // graceful shutdown
@ -160,12 +160,12 @@ func wait(c *cfg) {
go func() {
defer drain.Done()
for err := range c.internalErr {
c.log.Warn(logs.FrostFSNodeInternalApplicationError,
c.log.Warn(context.Background(), logs.FrostFSNodeInternalApplicationError,
zap.String("message", err.Error()))
}
}()
c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop)
c.log.Debug(context.Background(), logs.FrostFSNodeWaitingForAllProcessesToStop)
c.wg.Wait()

View file

@ -17,11 +17,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand"
"github.com/nspcc-dev/neo-go/pkg/core/block"
"github.com/nspcc-dev/neo-go/pkg/core/state"
"github.com/nspcc-dev/neo-go/pkg/neorpc/result"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/waiter"
"github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/vm/vmstate"
"go.uber.org/zap"
)
@ -39,20 +35,16 @@ func (c *cfg) initMorphComponents(ctx context.Context) {
lookupScriptHashesInNNS(c) // smart contract auto negotiation
if c.cfgMorph.notaryEnabled {
err := c.cfgMorph.client.EnableNotarySupport(
client.WithProxyContract(
c.cfgMorph.proxyScriptHash,
),
)
fatalOnErr(err)
}
c.log.Info(logs.FrostFSNodeNotarySupport,
zap.Bool("sidechain_enabled", c.cfgMorph.notaryEnabled),
err := c.cfgMorph.client.EnableNotarySupport(
client.WithProxyContract(
c.cfgMorph.proxyScriptHash,
),
)
fatalOnErr(err)
wrap, err := nmClient.NewFromMorph(c.cfgMorph.client, c.cfgNetmap.scriptHash, 0, nmClient.TryNotary())
c.log.Info(ctx, logs.FrostFSNodeNotarySupport)
wrap, err := nmClient.NewFromMorph(c.cfgMorph.client, c.cfgNetmap.scriptHash, 0)
fatalOnErr(err)
var netmapSource netmap.Source
@ -64,7 +56,7 @@ func (c *cfg) initMorphComponents(ctx context.Context) {
msPerBlock, err := c.cfgMorph.client.MsPerBlock()
fatalOnErr(err)
c.cfgMorph.cacheTTL = time.Duration(msPerBlock) * time.Millisecond
c.log.Debug(logs.FrostFSNodeMorphcacheTTLFetchedFromNetwork, zap.Duration("value", c.cfgMorph.cacheTTL))
c.log.Debug(ctx, logs.FrostFSNodeMorphcacheTTLFetchedFromNetwork, zap.Duration("value", c.cfgMorph.cacheTTL))
}
if c.cfgMorph.cacheTTL < 0 {
@ -102,41 +94,35 @@ func initMorphClient(ctx context.Context, c *cfg) {
client.WithDialerSource(c.dialerSource),
)
if err != nil {
c.log.Info(logs.FrostFSNodeFailedToCreateNeoRPCClient,
c.log.Info(ctx, logs.FrostFSNodeFailedToCreateNeoRPCClient,
zap.Any("endpoints", addresses),
zap.String("error", err.Error()),
zap.Error(err),
)
fatalOnErr(err)
}
c.onShutdown(func() {
c.log.Info(logs.FrostFSNodeClosingMorphComponents)
c.log.Info(ctx, logs.FrostFSNodeClosingMorphComponents)
cli.Close()
})
if err := cli.SetGroupSignerScope(); err != nil {
c.log.Info(logs.FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal, zap.Error(err))
c.log.Info(ctx, logs.FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal, zap.Error(err))
}
c.cfgMorph.client = cli
c.cfgMorph.notaryEnabled = cli.ProbeNotary()
}
func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) {
// skip notary deposit in non-notary environments
if !c.cfgMorph.notaryEnabled {
return
}
tx, vub, err := makeNotaryDeposit(c)
tx, vub, err := makeNotaryDeposit(ctx, c)
fatalOnErr(err)
if tx.Equals(util.Uint256{}) {
// non-error deposit with an empty TX hash means
// that the deposit has already been made; no
// need to wait it.
c.log.Info(logs.FrostFSNodeNotaryDepositHasAlreadyBeenMade)
c.log.Info(ctx, logs.FrostFSNodeNotaryDepositHasAlreadyBeenMade)
return
}
@ -144,7 +130,7 @@ func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) {
fatalOnErr(err)
}
func makeNotaryDeposit(c *cfg) (util.Uint256, uint32, error) {
func makeNotaryDeposit(ctx context.Context, c *cfg) (util.Uint256, uint32, error) {
const (
// gasMultiplier defines how many times more the notary
// balance must be compared to the GAS balance of the node:
@ -161,51 +147,16 @@ func makeNotaryDeposit(c *cfg) (util.Uint256, uint32, error) {
return util.Uint256{}, 0, fmt.Errorf("could not calculate notary deposit: %w", err)
}
return c.cfgMorph.client.DepositEndlessNotary(depositAmount)
}
var (
errNotaryDepositFail = errors.New("notary deposit tx has faulted")
errNotaryDepositTimeout = errors.New("notary deposit tx has not appeared in the network")
)
type waiterClient struct {
c *client.Client
}
func (w *waiterClient) Context() context.Context {
return context.Background()
}
func (w *waiterClient) GetApplicationLog(hash util.Uint256, trig *trigger.Type) (*result.ApplicationLog, error) {
return w.c.GetApplicationLog(hash, trig)
}
func (w *waiterClient) GetBlockCount() (uint32, error) {
return w.c.BlockCount()
}
func (w *waiterClient) GetVersion() (*result.Version, error) {
return w.c.GetVersion()
return c.cfgMorph.client.DepositEndlessNotary(ctx, depositAmount)
}
func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256, vub uint32) error {
w, err := waiter.NewPollingBased(&waiterClient{c: c.cfgMorph.client})
if err != nil {
return fmt.Errorf("could not create notary deposit waiter: %w", err)
if err := c.cfgMorph.client.WaitTxHalt(ctx, client.InvokeRes{Hash: tx, VUB: vub}); err != nil {
return err
}
res, err := w.WaitAny(ctx, vub, tx)
if err != nil {
if errors.Is(err, waiter.ErrTxNotAccepted) {
return errNotaryDepositTimeout
}
return fmt.Errorf("could not wait for notary deposit persists in chain: %w", err)
}
if res.Execution.VMState.HasFlag(vmstate.Halt) {
c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted)
return nil
}
return errNotaryDepositFail
c.log.Info(ctx, logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted)
return nil
}
func listenMorphNotifications(ctx context.Context, c *cfg) {
@ -217,7 +168,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) {
fromSideChainBlock, err := c.persistate.UInt32(persistateSideChainLastBlockKey)
if err != nil {
fromSideChainBlock = 0
c.log.Warn(logs.FrostFSNodeCantGetLastProcessedSideChainBlockNumber, zap.String("error", err.Error()))
c.log.Warn(ctx, logs.FrostFSNodeCantGetLastProcessedSideChainBlockNumber, zap.Error(err))
}
subs, err = subscriber.New(ctx, &subscriber.Params{
@ -246,7 +197,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) {
setNetmapNotificationParser(c, newEpochNotification, func(src *state.ContainedNotificationEvent) (event.Event, error) {
res, err := netmapEvent.ParseNewEpoch(src)
if err == nil {
c.log.Info(logs.FrostFSNodeNewEpochEventFromSidechain,
c.log.Info(ctx, logs.FrostFSNodeNewEpochEventFromSidechain,
zap.Uint64("number", res.(netmapEvent.NewEpoch).EpochNumber()),
)
}
@ -256,12 +207,12 @@ func listenMorphNotifications(ctx context.Context, c *cfg) {
registerNotificationHandlers(c.cfgNetmap.scriptHash, lis, c.cfgNetmap.parsers, c.cfgNetmap.subscribers)
registerNotificationHandlers(c.cfgContainer.scriptHash, lis, c.cfgContainer.parsers, c.cfgContainer.subscribers)
registerBlockHandler(lis, func(block *block.Block) {
c.log.Debug(logs.FrostFSNodeNewBlock, zap.Uint32("index", block.Index))
registerBlockHandler(lis, func(ctx context.Context, block *block.Block) {
c.log.Debug(ctx, logs.FrostFSNodeNewBlock, zap.Uint32("index", block.Index))
err = c.persistate.SetUInt32(persistateSideChainLastBlockKey, block.Index)
if err != nil {
c.log.Warn(logs.FrostFSNodeCantUpdatePersistentState,
c.log.Warn(ctx, logs.FrostFSNodeCantUpdatePersistentState,
zap.String("chain", "side"),
zap.Uint32("block_index", block.Index))
}
@ -272,27 +223,17 @@ func registerNotificationHandlers(scHash util.Uint160, lis event.Listener, parse
subs map[event.Type][]event.Handler,
) {
for typ, handlers := range subs {
pi := event.NotificationParserInfo{}
pi.SetType(typ)
pi.SetScriptHash(scHash)
p, ok := parsers[typ]
if !ok {
panic(fmt.Sprintf("missing parser for event %s", typ))
}
pi.SetParser(p)
lis.SetNotificationParser(pi)
for _, h := range handlers {
hi := event.NotificationHandlerInfo{}
hi.SetType(typ)
hi.SetScriptHash(scHash)
hi.SetHandler(h)
lis.RegisterNotificationHandler(hi)
}
lis.RegisterNotificationHandler(event.NotificationHandlerInfo{
Contract: scHash,
Type: typ,
Parser: p,
Handlers: handlers,
})
}
}
@ -321,10 +262,6 @@ func lookupScriptHashesInNNS(c *cfg) {
)
for _, t := range targets {
if t.nnsName == client.NNSProxyContractName && !c.cfgMorph.notaryEnabled {
continue // ignore proxy contract if notary disabled
}
if emptyHash.Equals(*t.h) {
*t.h, err = c.cfgMorph.client.NNSContractAddress(t.nnsName)
fatalOnErrDetails(fmt.Sprintf("can't resolve %s in NNS", t.nnsName), err)

View file

@ -145,7 +145,7 @@ func initNetmapService(ctx context.Context, c *cfg) {
c.initMorphComponents(ctx)
initNetmapState(c)
initNetmapState(ctx, c)
server := netmapTransportGRPC.New(
netmapService.NewSignService(
@ -166,51 +166,52 @@ func initNetmapService(ctx context.Context, c *cfg) {
c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) {
netmapGRPC.RegisterNetmapServiceServer(s, server)
// TODO(@aarifullin): #1487 remove the dual service support.
s.RegisterService(frostFSServiceDesc(netmapGRPC.NetmapService_ServiceDesc), server)
})
addNewEpochNotificationHandlers(c)
}
func addNewEpochNotificationHandlers(c *cfg) {
addNewEpochNotificationHandler(c, func(ev event.Event) {
addNewEpochNotificationHandler(c, func(_ context.Context, ev event.Event) {
c.cfgNetmap.state.setCurrentEpoch(ev.(netmapEvent.NewEpoch).EpochNumber())
})
addNewEpochAsyncNotificationHandler(c, func(ev event.Event) {
addNewEpochAsyncNotificationHandler(c, func(ctx context.Context, ev event.Event) {
e := ev.(netmapEvent.NewEpoch).EpochNumber()
c.updateContractNodeInfo(e)
c.updateContractNodeInfo(ctx, e)
if !c.needBootstrap() || c.cfgNetmap.reBoostrapTurnedOff.Load() { // fixes #470
return
}
if err := c.bootstrap(); err != nil {
c.log.Warn(logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err))
if err := c.bootstrap(ctx); err != nil {
c.log.Warn(ctx, logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err))
}
})
if c.cfgMorph.notaryEnabled {
addNewEpochAsyncNotificationHandler(c, func(_ event.Event) {
_, _, err := makeNotaryDeposit(c)
if err != nil {
c.log.Error(logs.FrostFSNodeCouldNotMakeNotaryDeposit,
zap.String("error", err.Error()),
)
}
})
}
addNewEpochAsyncNotificationHandler(c, func(ctx context.Context, _ event.Event) {
_, _, err := makeNotaryDeposit(ctx, c)
if err != nil {
c.log.Error(ctx, logs.FrostFSNodeCouldNotMakeNotaryDeposit,
zap.Error(err),
)
}
})
}
// bootstrapNode adds current node to the Network map.
// Must be called after initNetmapService.
func bootstrapNode(c *cfg) {
func bootstrapNode(ctx context.Context, c *cfg) {
if c.needBootstrap() {
if c.IsMaintenance() {
c.log.Info(logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap)
c.log.Info(ctx, logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap)
return
}
err := c.bootstrap()
err := c.bootstrap(ctx)
fatalOnErrDetails("bootstrap error", err)
}
}
@ -237,17 +238,17 @@ func setNetmapNotificationParser(c *cfg, sTyp string, p event.NotificationParser
// initNetmapState inits current Network map state.
// Must be called after Morph components initialization.
func initNetmapState(c *cfg) {
func initNetmapState(ctx context.Context, c *cfg) {
epoch, err := c.cfgNetmap.wrapper.Epoch()
fatalOnErrDetails("could not initialize current epoch number", err)
var ni *netmapSDK.NodeInfo
ni, err = c.netmapInitLocalNodeState(epoch)
ni, err = c.netmapInitLocalNodeState(ctx, epoch)
fatalOnErrDetails("could not init network state", err)
stateWord := nodeState(ni)
c.log.Info(logs.FrostFSNodeInitialNetworkState,
c.log.Info(ctx, logs.FrostFSNodeInitialNetworkState,
zap.Uint64("epoch", epoch),
zap.String("state", stateWord),
)
@ -276,7 +277,7 @@ func nodeState(ni *netmapSDK.NodeInfo) string {
return "undefined"
}
func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) {
func (c *cfg) netmapInitLocalNodeState(ctx context.Context, epoch uint64) (*netmapSDK.NodeInfo, error) {
nmNodes, err := c.cfgNetmap.wrapper.GetCandidates()
if err != nil {
return nil, err
@ -304,7 +305,7 @@ func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error
if nmState != candidateState {
// This happens when the node was switched to maintenance without epoch tick.
// We expect it to continue staying in maintenance.
c.log.Info(logs.CandidateStatusPriority,
c.log.Info(ctx, logs.CandidateStatusPriority,
zap.String("netmap", nmState),
zap.String("candidate", candidateState))
}
@ -350,16 +351,16 @@ func addNewEpochAsyncNotificationHandler(c *cfg, h event.Handler) {
var errRelayBootstrap = errors.New("setting netmap status is forbidden in relay mode")
func (c *cfg) SetNetmapStatus(st control.NetmapStatus) error {
func (c *cfg) SetNetmapStatus(ctx context.Context, st control.NetmapStatus) error {
switch st {
default:
return fmt.Errorf("unsupported status %v", st)
case control.NetmapStatus_MAINTENANCE:
return c.setMaintenanceStatus(false)
return c.setMaintenanceStatus(ctx, false)
case control.NetmapStatus_ONLINE, control.NetmapStatus_OFFLINE:
}
c.stopMaintenance()
c.stopMaintenance(ctx)
if !c.needBootstrap() {
return errRelayBootstrap
@ -367,12 +368,12 @@ func (c *cfg) SetNetmapStatus(st control.NetmapStatus) error {
if st == control.NetmapStatus_ONLINE {
c.cfgNetmap.reBoostrapTurnedOff.Store(false)
return bootstrapOnline(c)
return bootstrapOnline(ctx, c)
}
c.cfgNetmap.reBoostrapTurnedOff.Store(true)
return c.updateNetMapState(func(*nmClient.UpdatePeerPrm) {})
return c.updateNetMapState(ctx, func(*nmClient.UpdatePeerPrm) {})
}
func (c *cfg) GetNetmapStatus() (control.NetmapStatus, uint64, error) {
@ -384,11 +385,11 @@ func (c *cfg) GetNetmapStatus() (control.NetmapStatus, uint64, error) {
return st, epoch, nil
}
func (c *cfg) ForceMaintenance() error {
return c.setMaintenanceStatus(true)
func (c *cfg) ForceMaintenance(ctx context.Context) error {
return c.setMaintenanceStatus(ctx, true)
}
func (c *cfg) setMaintenanceStatus(force bool) error {
func (c *cfg) setMaintenanceStatus(ctx context.Context, force bool) error {
netSettings, err := c.cfgNetmap.wrapper.ReadNetworkConfiguration()
if err != nil {
err = fmt.Errorf("read network settings to check maintenance allowance: %w", err)
@ -397,10 +398,10 @@ func (c *cfg) setMaintenanceStatus(force bool) error {
}
if err == nil || force {
c.startMaintenance()
c.startMaintenance(ctx)
if err == nil {
err = c.updateNetMapState((*nmClient.UpdatePeerPrm).SetMaintenance)
err = c.updateNetMapState(ctx, (*nmClient.UpdatePeerPrm).SetMaintenance)
}
if err != nil {
@ -413,13 +414,16 @@ func (c *cfg) setMaintenanceStatus(force bool) error {
// calls UpdatePeerState operation of Netmap contract's client for the local node.
// State setter is used to specify node state to switch to.
func (c *cfg) updateNetMapState(stateSetter func(*nmClient.UpdatePeerPrm)) error {
func (c *cfg) updateNetMapState(ctx context.Context, stateSetter func(*nmClient.UpdatePeerPrm)) error {
var prm nmClient.UpdatePeerPrm
prm.SetKey(c.key.PublicKey().Bytes())
stateSetter(&prm)
_, err := c.cfgNetmap.wrapper.UpdatePeerState(prm)
return err
res, err := c.cfgNetmap.wrapper.UpdatePeerState(ctx, prm)
if err != nil {
return err
}
return c.cfgNetmap.wrapper.Morph().WaitTxHalt(ctx, res)
}
type netInfo struct {

View file

@ -2,7 +2,6 @@ package main
import (
"context"
"errors"
"fmt"
"net"
@ -14,12 +13,9 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine"
morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/cache"
objectTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/object/grpc"
objectService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl"
v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2"
objectAPE "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/ape"
objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer"
@ -38,8 +34,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -63,19 +57,19 @@ type objectSvc struct {
func (c *cfg) MaxObjectSize() uint64 {
sz, err := c.cfgNetmap.wrapper.MaxObjectSize()
if err != nil {
c.log.Error(logs.FrostFSNodeCouldNotGetMaxObjectSizeValue,
zap.String("error", err.Error()),
c.log.Error(context.Background(), logs.FrostFSNodeCouldNotGetMaxObjectSizeValue,
zap.Error(err),
)
}
return sz
}
func (s *objectSvc) Put() (objectService.PutObjectStream, error) {
func (s *objectSvc) Put(_ context.Context) (objectService.PutObjectStream, error) {
return s.put.Put()
}
func (s *objectSvc) Patch() (objectService.PatchObjectStream, error) {
func (s *objectSvc) Patch(_ context.Context) (objectService.PatchObjectStream, error) {
return s.patch.Patch()
}
@ -142,24 +136,6 @@ func (fn *innerRingFetcherWithNotary) InnerRingKeys() ([][]byte, error) {
return result, nil
}
type innerRingFetcherWithoutNotary struct {
nm *nmClient.Client
}
func (f *innerRingFetcherWithoutNotary) InnerRingKeys() ([][]byte, error) {
keys, err := f.nm.GetInnerRingList()
if err != nil {
return nil, fmt.Errorf("can't get inner ring keys from netmap contract: %w", err)
}
result := make([][]byte, 0, len(keys))
for i := range keys {
result = append(result, keys[i].Bytes())
}
return result, nil
}
func initObjectService(c *cfg) {
keyStorage := util.NewKeyStorage(&c.key.PrivateKey, c.privateTokenStore, c.cfgNetmap.state)
@ -220,12 +196,15 @@ func initObjectService(c *cfg) {
c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) {
objectGRPC.RegisterObjectServiceServer(s, server)
// TODO(@aarifullin): #1487 remove the dual service support.
s.RegisterService(frostFSServiceDesc(objectGRPC.ObjectService_ServiceDesc), server)
})
}
func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.ClientCache) {
if policerconfig.UnsafeDisable(c.appCfg) {
c.log.Warn(logs.FrostFSNodePolicerIsDisabled)
c.log.Warn(context.Background(), logs.FrostFSNodePolicerIsDisabled)
return
}
@ -289,8 +268,8 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl
_, err := ls.Inhume(ctx, inhumePrm)
if err != nil {
c.log.Warn(logs.FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage,
zap.String("error", err.Error()),
c.log.Warn(ctx, logs.FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage,
zap.Error(err),
)
}
}),
@ -307,13 +286,8 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl
}
func createInnerRingFetcher(c *cfg) v2.InnerRingFetcher {
if c.cfgMorph.client.ProbeNotary() {
return &innerRingFetcherWithNotary{
sidechain: c.cfgMorph.client,
}
}
return &innerRingFetcherWithoutNotary{
nm: c.cfgNetmap.wrapper,
return &innerRingFetcherWithNotary{
sidechain: c.cfgMorph.client,
}
}
@ -458,17 +432,10 @@ func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Servi
}
func createACLServiceV2(c *cfg, apeSvc *objectAPE.Service, irFetcher *cachedIRFetcher) v2.Service {
ls := c.cfgObject.cfgLocalStorage.localStorage
return v2.New(
apeSvc,
c.netMapSource,
irFetcher,
acl.NewChecker(
c.cfgNetmap.state,
c.cfgObject.eaclSource,
eaclSDK.NewValidator(),
ls),
c.cfgObject.cnrSource,
v2.WithLogger(c.log),
)
@ -490,29 +457,6 @@ func createAPEService(c *cfg, splitSvc *objectService.TransportSplitter) *object
)
}
type morphEACLFetcher struct {
w *cntClient.Client
}
func (s *morphEACLFetcher) GetEACL(cnr cid.ID) (*containercore.EACL, error) {
eaclInfo, err := s.w.GetEACL(cnr)
if err != nil {
return nil, err
}
binTable, err := eaclInfo.Value.Marshal()
if err != nil {
return nil, fmt.Errorf("marshal eACL table: %w", err)
}
if !eaclInfo.Signature.Verify(binTable) {
// TODO(@cthulhu-rider): #468 use "const" error
return nil, errors.New("invalid signature of the eACL table")
}
return eaclInfo, nil
}
type engineWithoutNotifications struct {
engine *engine.StorageEngine
}

View file

@ -1,17 +1,18 @@
package main
import (
"context"
"runtime"
profilerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/profiler"
httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
)
func initProfilerService(c *cfg) {
func initProfilerService(ctx context.Context, c *cfg) {
tuneProfilers(c)
pprof, _ := pprofComponent(c)
pprof.init(c)
pprof.init(ctx, c)
}
func pprofComponent(c *cfg) (*httpComponent, bool) {

Some files were not shown because too many files have changed in this diff Show more