diff --git a/.docker/Dockerfile.adm b/.docker/Dockerfile.adm index 09d66e642..f0075c7ab 100644 --- a/.docker/Dockerfile.adm +++ b/.docker/Dockerfile.adm @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ci b/.docker/Dockerfile.ci index 010616a17..e22762b7b 100644 --- a/.docker/Dockerfile.ci +++ b/.docker/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM golang:1.19 +FROM golang:1.20 WORKDIR /tmp diff --git a/.docker/Dockerfile.cli b/.docker/Dockerfile.cli index c706359b3..8fad863be 100644 --- a/.docker/Dockerfile.cli +++ b/.docker/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ir b/.docker/Dockerfile.ir index 9f8e72386..0b01ea7e4 100644 --- a/.docker/Dockerfile.ir +++ b/.docker/Dockerfile.ir @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage b/.docker/Dockerfile.storage index 39eb19559..dd9ff2684 100644 --- a/.docker/Dockerfile.storage +++ b/.docker/Dockerfile.storage @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage-testnet b/.docker/Dockerfile.storage-testnet deleted file mode 100644 index 908ff0aad..000000000 --- a/.docker/Dockerfile.storage-testnet +++ /dev/null @@ -1,19 +0,0 @@ -FROM golang:1.18 as builder -ARG BUILD=now -ARG VERSION=dev -ARG REPO=repository -WORKDIR /src -COPY . /src - -RUN make bin/frostfs-node - -# Executable image -FROM alpine AS frostfs-node -RUN apk add --no-cache bash - -WORKDIR / - -COPY --from=builder /src/bin/frostfs-node /bin/frostfs-node -COPY --from=builder /src/config/testnet/config.yml /config.yml - -CMD ["frostfs-node", "--config", "/config.yml"] diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml new file mode 100644 index 000000000..5e4a97b93 --- /dev/null +++ b/.forgejo/workflows/build.yml @@ -0,0 +1,38 @@ +name: Build + +on: [pull_request] + +jobs: + build: + name: Build Components + runs-on: ubuntu-latest + strategy: + matrix: + go_versions: [ '1.19', '1.20' ] + + steps: + - uses: actions/checkout@v3 + with: + # Allows to fetch all history for all branches and tags. + # Need this for proper versioning. + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '${{ matrix.go_versions }}' + + - name: Build CLI + run: make bin/frostfs-cli + + - name: Build NODE + run: make bin/frostfs-node + + - name: Build IR + run: make bin/frostfs-ir + + - name: Build ADM + run: make bin/frostfs-adm + + - name: Build LENS + run: make bin/frostfs-lens diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml new file mode 100644 index 000000000..13395a64b --- /dev/null +++ b/.forgejo/workflows/tests.yml @@ -0,0 +1,72 @@ +name: Tests and linters +on: [pull_request] + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + cache: true + + - name: golangci-lint + uses: https://github.com/golangci/golangci-lint-action@v3 + with: + version: latest + + tests: + name: Tests + runs-on: ubuntu-latest + strategy: + matrix: + go_versions: [ '1.19', '1.20' ] + fail-fast: false + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '${{ matrix.go_versions }}' + cache: true + + - name: Run tests + run: make test + + tests-race: + name: Tests with -race + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + cache: true + + - name: Run tests + run: go test ./... -count=1 -race + + staticcheck: + name: Staticcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + cache: true + + - name: Install staticcheck + run: make staticcheck-install + + - name: Run staticcheck + run: make staticcheck-run diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml new file mode 100644 index 000000000..9390ad2d6 --- /dev/null +++ b/.forgejo/workflows/vulncheck.yml @@ -0,0 +1,22 @@ +name: Vulncheck +on: [pull_request] + +jobs: + vulncheck: + name: Vulncheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + + - name: Install govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@latest + + - name: Run govulncheck + run: govulncheck ./... diff --git a/.golangci.yml b/.golangci.yml index fae355a3d..643390075 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,7 +4,7 @@ # options for analysis running run: # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 10m + timeout: 20m # include test files or not, default is true tests: false @@ -31,6 +31,12 @@ linters-settings: statements: 60 # default 40 gocognit: min-complexity: 40 # default 30 + importas: + no-unaliased: true + no-extra-aliases: false + alias: + pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object + alias: objectSDK linters: enable: @@ -62,5 +68,6 @@ linters: - funlen - gocognit - contextcheck + - importas disable-all: true fast: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ed434da5..6afb8f700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,17 +6,44 @@ Changelog for FrostFS Node ### Added - Support impersonate bearer token (#229) - Change log level on SIGHUP for ir (#125) +- Reload pprof and metrics on SIGHUP for ir (#125) +- Support copies number parameter in `frostfs-cli object put` (#351) +- Set extra wallets on SIGHUP for ir (#125) +- Writecache metrics (#312) +- Add tree service metrics (#370) ### Changed +- `frostfs-cli util locode generate` is now much faster (#309) ### Fixed - Take network settings into account during netmap contract update (#100) - Read config files from dir even if config file not provided via `--config` for node (#238) - Notary requests parsing according to `neo-go`'s updates (#268) - Tree service panic in its internal client cache (#322) - Iterate over endpoints when create ws client in morph's constructor (#304) +- Delete complex objects with GC (#332) ### Removed ### Updated +- `neo-go` to `v0.101.1` +- `google.golang.org/grpc` to `v1.55.0` +- `paulmach/orb` to `v0.9.2` +- `go.etcd.io/bbolt` to `v1.3.7` +- `github.com/nats-io/nats.go` to `v1.25.0` +- `golang.org/x/sync` to `v0.2.0` +- `golang.org/x/term` to `v0.8.0` +- `github.com/spf13/cobra` to `v1.7.0` +- `github.com/panjf2000/ants/v2` `v2.7.4` +- `github.com/multiformats/go-multiaddr` to `v0.9.0` +- `github.com/hashicorp/golang-lru/v2` to `v2.0.2` +- `go.uber.org/atomic` to `v1.11.0` +- Minimum go version to v1.19 +- `github.com/prometheus/client_golang` to `v1.15.1` +- `github.com/prometheus/client_model` to `v0.4.0` +- `go.opentelemetry.io/otel` to `v1.15.1` +- `go.opentelemetry.io/otel/trace` to `v1.15.1` +- `github.com/spf13/cast` to `v1.5.1` +- `git.frostfs.info/TrueCloudLab/hrw` to `v1.2.1` + ### Updating from v0.36.0 ## [v0.36.0] - 2023-04-12 - Furtwängler diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce617f7f6..53ff7c8df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ First, thank you for contributing! We love and encourage pull requests from everyone. Please follow the guidelines: -- Check the open [issues](https://github.com/TrueCloudLab/frostfs-node/issues) and - [pull requests](https://github.com/TrueCloudLab/frostfs-node/pulls) for existing +- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-node/issues) and + [pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-node/pulls) for existing discussions. - Open an issue first, to discuss a new feature or enhancement. @@ -27,19 +27,19 @@ Start by forking the `frostfs-node` repository, make changes in a branch and the send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: -### Set up your GitHub Repository -Fork [FrostFS node upstream](https://github.com/TrueCloudLab/frostfs-node/fork) source +### Set up your Forgejo repository +Fork [FrostFS node upstream](https://git.frostfs.info/TrueCloudLab/frostfs-node) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). ```sh -$ git clone https://github.com/TrueCloudLab/frostfs-node +$ git clone https://git.frostfs.info/TrueCloudLab/frostfs-node ``` ### Set up git remote as ``upstream`` ```sh $ cd frostfs-node -$ git remote add upstream https://github.com/TrueCloudLab/frostfs-node +$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-node $ git fetch upstream $ git merge upstream/master ... @@ -58,7 +58,7 @@ $ git checkout -b feature/123-something_awesome After your code changes, make sure - To add test cases for the new code. -- To run `make lint` +- To run `make lint` and `make staticcheck-run` - To squash your commits into a single commit or a series of logically separated commits run `git rebase -i`. It's okay to force update your pull request. - To run `make test` and `make all` completes. @@ -89,8 +89,8 @@ $ git push origin feature/123-something_awesome ``` ### Create a Pull Request -Pull requests can be created via GitHub. Refer to [this -document](https://help.github.com/articles/creating-a-pull-request/) for +Pull requests can be created via Forgejo. Refer to [this +document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged. diff --git a/Makefile b/Makefile index 29625e657..df53b8772 100755 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ image-%: -t $(HUB_IMAGE)-$*:$(HUB_TAG) . # Build all Docker images -images: image-storage image-ir image-cli image-adm image-storage-testnet +images: image-storage image-ir image-cli image-adm # Build dirty local Docker images dirty-images: image-dirty-storage image-dirty-ir image-dirty-cli image-dirty-adm @@ -126,7 +126,7 @@ imports: # Run Unit Test with go test test: @echo "⇒ Running go test" - @go test ./... + @go test ./... -count=1 pre-commit-run: @pre-commit run -a --hook-stage manual @@ -135,8 +135,12 @@ pre-commit-run: lint: @golangci-lint --timeout=5m run +# Install staticcheck +staticcheck-install: + @go install honnef.co/go/tools/cmd/staticcheck@latest + # Run staticcheck -staticcheck: +staticcheck-run: @staticcheck ./... # Run linters in Docker diff --git a/README.md b/README.md index 81701c441..f228cd426 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract # Building -To make all binaries you need Go 1.18+ and `make`: +To make all binaries you need Go 1.19+ and `make`: ``` make all ``` diff --git a/cmd/frostfs-adm/docs/deploy.md b/cmd/frostfs-adm/docs/deploy.md index aead65fe0..974c2a93c 100644 --- a/cmd/frostfs-adm/docs/deploy.md +++ b/cmd/frostfs-adm/docs/deploy.md @@ -18,6 +18,7 @@ To start a network, you need a set of consensus nodes, the same number of Alphabet nodes and any number of Storage nodes. While the number of Storage nodes can be scaled almost infinitely, the number of consensus and Alphabet nodes can't be changed so easily right now. Consider this before going any further. +Note also that there is an upper limit on the number of alphabet nodes (currently 22). It is easier to use`frostfs-adm` with a predefined configuration. First, create a network configuration file. In this example, there is going to be only one diff --git a/cmd/frostfs-adm/internal/modules/morph/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance.go index f97250c38..6debc50b9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance.go @@ -37,11 +37,6 @@ const ( dumpBalancesAlphabetFlag = "alphabet" dumpBalancesProxyFlag = "proxy" dumpBalancesUseScriptHashFlag = "script-hash" - - // notaryEnabled signifies whether contracts were deployed in a notary-enabled environment. - // The setting is here to simplify testing and building the command for testnet (notary currently disabled). - // It will be removed eventually. - notaryEnabled = true ) func dumpBalances(cmd *cobra.Command, _ []string) error { @@ -60,7 +55,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { inv := invoker.New(c, nil) - if !notaryEnabled || dumpStorage || dumpAlphabet || dumpProxy { + if dumpStorage || dumpAlphabet || dumpProxy { nnsCs, err = c.GetContractStateByID(1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) @@ -72,7 +67,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { } } - irList, err := fetchIRNodes(c, nmHash, rolemgmt.Hash) + irList, err := fetchIRNodes(c, rolemgmt.Hash) if err != nil { return err } @@ -187,40 +182,22 @@ func printAlphabetContractBalances(cmd *cobra.Command, c Client, inv *invoker.In return nil } -func fetchIRNodes(c Client, nmHash, desigHash util.Uint160) ([]accBalancePair, error) { - var irList []accBalancePair - +func fetchIRNodes(c Client, desigHash util.Uint160) ([]accBalancePair, error) { inv := invoker.New(c, nil) - if notaryEnabled { - height, err := c.GetBlockCount() - if err != nil { - return nil, fmt.Errorf("can't get block height: %w", err) - } + height, err := c.GetBlockCount() + if err != nil { + return nil, fmt.Errorf("can't get block height: %w", err) + } - arr, err := getDesignatedByRole(inv, desigHash, noderoles.NeoFSAlphabet, height) - if err != nil { - return nil, errors.New("can't fetch list of IR nodes from the netmap contract") - } + arr, err := getDesignatedByRole(inv, desigHash, noderoles.NeoFSAlphabet, height) + if err != nil { + return nil, errors.New("can't fetch list of IR nodes from the netmap contract") + } - irList = make([]accBalancePair, len(arr)) - for i := range arr { - irList[i].scriptHash = arr[i].GetScriptHash() - } - } else { - arr, err := unwrap.ArrayOfBytes(inv.Call(nmHash, "innerRingList")) - if err != nil { - return nil, errors.New("can't fetch list of IR nodes from the netmap contract") - } - - irList = make([]accBalancePair, len(arr)) - for i := range arr { - pub, err := keys.NewPublicKeyFromBytes(arr[i], elliptic.P256()) - if err != nil { - return nil, fmt.Errorf("can't parse IR node public key: %w", err) - } - irList[i].scriptHash = pub.GetScriptHash() - } + irList := make([]accBalancePair, len(arr)) + for i := range arr { + irList[i].scriptHash = arr[i].GetScriptHash() } return irList, nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index e714482dd..ccdc4519f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -21,6 +21,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" "github.com/spf13/viper" + "golang.org/x/sync/errgroup" ) const ( @@ -38,6 +39,9 @@ func generateAlphabetCreds(cmd *cobra.Command, _ []string) error { if size == 0 { return errors.New("size must be > 0") } + if size > maxAlphabetNodes { + return ErrTooManyAlphabetNodes + } v := viper.GetViper() walletDir := config.ResolveHomePath(viper.GetString(alphabetWalletsFlag)) @@ -92,28 +96,32 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er pubs[i] = w.Accounts[0].PrivateKey().PublicKey() } + var errG errgroup.Group + // Create committee account with N/2+1 multi-signature. majCount := smartcontract.GetMajorityHonestNodeCount(size) - for i, w := range wallets { - if err := addMultisigAccount(w, majCount, committeeAccountName, passwords[i], pubs); err != nil { - return nil, fmt.Errorf("can't create committee account: %w", err) - } - } - // Create consensus account with 2*N/3+1 multi-signature. bftCount := smartcontract.GetDefaultHonestNodeCount(size) - for i, w := range wallets { - if err := addMultisigAccount(w, bftCount, consensusAccountName, passwords[i], pubs); err != nil { - return nil, fmt.Errorf("can't create consensus account: %w", err) - } + for i := range wallets { + i := i + ps := make(keys.PublicKeys, len(pubs)) + copy(ps, pubs) + errG.Go(func() error { + if err := addMultisigAccount(wallets[i], majCount, committeeAccountName, passwords[i], ps); err != nil { + return fmt.Errorf("can't create committee account: %w", err) + } + if err := addMultisigAccount(wallets[i], bftCount, consensusAccountName, passwords[i], ps); err != nil { + return fmt.Errorf("can't create consentus account: %w", err) + } + if err := wallets[i].SavePretty(); err != nil { + return fmt.Errorf("can't save wallet: %w", err) + } + return nil + }) } - - for _, w := range wallets { - if err := w.SavePretty(); err != nil { - return nil, fmt.Errorf("can't save wallet: %w", err) - } + if err := errG.Wait(); err != nil { + return nil, err } - return passwords, nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate_test.go index 39cfc5718..457813df0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strconv" + "sync" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" @@ -71,24 +72,31 @@ func TestGenerateAlphabet(t *testing.T) { buf.WriteString(testContractPassword + "\r") require.NoError(t, generateAlphabetCreds(generateAlphabetCmd, nil)) + var wg sync.WaitGroup for i := uint64(0); i < size; i++ { - p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") - w, err := wallet.NewWalletFromFile(p) - require.NoError(t, err, "wallet doesn't exist") - require.Equal(t, 3, len(w.Accounts), "not all accounts were created") - for _, a := range w.Accounts { - err := a.Decrypt(strconv.FormatUint(i, 10), keys.NEP2ScryptParams()) - require.NoError(t, err, "can't decrypt account") - switch a.Label { - case consensusAccountName: - require.Equal(t, smartcontract.GetDefaultHonestNodeCount(size), len(a.Contract.Parameters)) - case committeeAccountName: - require.Equal(t, smartcontract.GetMajorityHonestNodeCount(size), len(a.Contract.Parameters)) - default: - require.Equal(t, singleAccountName, a.Label) + i := i + go func() { + defer wg.Done() + p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") + w, err := wallet.NewWalletFromFile(p) + require.NoError(t, err, "wallet doesn't exist") + require.Equal(t, 3, len(w.Accounts), "not all accounts were created") + + for _, a := range w.Accounts { + err := a.Decrypt(strconv.FormatUint(i, 10), keys.NEP2ScryptParams()) + require.NoError(t, err, "can't decrypt account") + switch a.Label { + case consensusAccountName: + require.Equal(t, smartcontract.GetDefaultHonestNodeCount(size), len(a.Contract.Parameters)) + case committeeAccountName: + require.Equal(t, smartcontract.GetMajorityHonestNodeCount(size), len(a.Contract.Parameters)) + default: + require.Equal(t, singleAccountName, a.Label) + } } - } + }() } + wg.Wait() t.Run("check contract group wallet", func(t *testing.T) { p := filepath.Join(walletDir, contractWalletFilename) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 494ad5296..dec1fba20 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -23,6 +23,13 @@ import ( "github.com/spf13/viper" ) +const ( + // maxAlphabetNodes is the maximum number of candidates allowed, which is currently limited by the size + // of the invocation script. + // See: https://github.com/nspcc-dev/neo-go/blob/740488f7f35e367eaa99a71c0a609c315fe2b0fc/pkg/core/transaction/witness.go#L10 + maxAlphabetNodes = 22 +) + type cache struct { nnsCs *state.Contract groupKey *keys.PublicKey @@ -45,6 +52,8 @@ type initializeContext struct { ContractPath string } +var ErrTooManyAlphabetNodes = fmt.Errorf("too many alphabet nodes (maximum allowed is %d)", maxAlphabetNodes) + func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { initCtx, err := newInitializeContext(cmd, viper.GetViper()) if err != nil { @@ -111,6 +120,10 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex return nil, err } + if len(wallets) > maxAlphabetNodes { + return nil, ErrTooManyAlphabetNodes + } + needContracts := cmd.Name() == "update-contracts" || cmd.Name() == "init" var w *wallet.Wallet @@ -197,11 +210,11 @@ func validateInit(cmd *cobra.Command) error { func createClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (Client, error) { var c Client var err error - if v.GetString(localDumpFlag) != "" { - if v.GetString(endpointFlag) != "" { + if ldf := cmd.Flags().Lookup(localDumpFlag); ldf != nil && ldf.Changed { + if cmd.Flags().Changed(endpointFlag) { return nil, fmt.Errorf("`%s` and `%s` flags are mutually exclusive", endpointFlag, localDumpFlag) } - c, err = newLocalClient(cmd, v, wallets) + c, err = newLocalClient(cmd, v, wallets, ldf.Value.String()) } else { c, err = getN3Client(v) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index b1542cc92..469b269de 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -19,33 +19,24 @@ import ( ) // initialAlphabetNEOAmount represents the total amount of GAS distributed between alphabet nodes. -const initialAlphabetNEOAmount = native.NEOTotalSupply - -func (c *initializeContext) registerCandidates() error { - neoHash := neo.Hash - - cc, err := unwrap.Array(c.ReadOnlyInvoker.Call(neoHash, "getCandidates")) - if err != nil { - return fmt.Errorf("`getCandidates`: %w", err) - } - - if len(cc) > 0 { - c.Command.Println("Candidates are already registered.") - return nil - } +const ( + initialAlphabetNEOAmount = native.NEOTotalSupply + registerBatchSize = transaction.MaxAttributes - 1 +) +func (c *initializeContext) registerCandidateRange(start, end int) error { regPrice, err := c.getCandidateRegisterPrice() if err != nil { return fmt.Errorf("can't fetch registration price: %w", err) } w := io.NewBufBinWriter() - emit.AppCall(w.BinWriter, neoHash, "setRegisterPrice", callflag.States, 1) - for _, acc := range c.Accounts { - emit.AppCall(w.BinWriter, neoHash, "registerCandidate", callflag.States, acc.PrivateKey().PublicKey().Bytes()) + emit.AppCall(w.BinWriter, neo.Hash, "setRegisterPrice", callflag.States, 1) + for _, acc := range c.Accounts[start:end] { + emit.AppCall(w.BinWriter, neo.Hash, "registerCandidate", callflag.States, acc.PrivateKey().PublicKey().Bytes()) emit.Opcodes(w.BinWriter, opcode.ASSERT) } - emit.AppCall(w.BinWriter, neoHash, "setRegisterPrice", callflag.States, regPrice) + emit.AppCall(w.BinWriter, neo.Hash, "setRegisterPrice", callflag.States, regPrice) if w.Err != nil { panic(fmt.Sprintf("BUG: %v", w.Err)) } @@ -54,14 +45,14 @@ func (c *initializeContext) registerCandidates() error { Signer: c.getSigner(false, c.CommitteeAcc), Account: c.CommitteeAcc, }} - for i := range c.Accounts { + for _, acc := range c.Accounts[start:end] { signers = append(signers, rpcclient.SignerAccount{ Signer: transaction.Signer{ - Account: c.Accounts[i].Contract.ScriptHash(), + Account: acc.Contract.ScriptHash(), Scopes: transaction.CustomContracts, - AllowedContracts: []util.Uint160{neoHash}, + AllowedContracts: []util.Uint160{neo.Hash}, }, - Account: c.Accounts[i], + Account: acc, }) } @@ -74,8 +65,8 @@ func (c *initializeContext) registerCandidates() error { } network := c.CommitteeAct.GetNetwork() - for i := range c.Accounts { - if err := c.Accounts[i].SignTx(network, tx); err != nil { + for _, acc := range c.Accounts[start:end] { + if err := acc.SignTx(network, tx); err != nil { return fmt.Errorf("can't sign a transaction: %w", err) } } @@ -83,6 +74,39 @@ func (c *initializeContext) registerCandidates() error { return c.sendTx(tx, c.Command, true) } +func (c *initializeContext) registerCandidates() error { + cc, err := unwrap.Array(c.ReadOnlyInvoker.Call(neo.Hash, "getCandidates")) + if err != nil { + return fmt.Errorf("`getCandidates`: %w", err) + } + + need := len(c.Accounts) + have := len(cc) + + if need == have { + c.Command.Println("Candidates are already registered.") + return nil + } + + // Register candidates in batches in order to overcome the signers amount limit. + // See: https://github.com/nspcc-dev/neo-go/blob/master/pkg/core/transaction/transaction.go#L27 + for i := 0; i < need; i += registerBatchSize { + start, end := i, i+registerBatchSize + if end > need { + end = need + } + // This check is sound because transactions are accepted/rejected atomically. + if have >= end { + continue + } + if err := c.registerCandidateRange(start, end); err != nil { + return fmt.Errorf("registering candidates %d..%d: %q", start, end-1, err) + } + } + + return nil +} + func (c *initializeContext) transferNEOToAlphabetContracts() error { neoHash := neo.Hash diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index fb2dc3e3f..30a7168dd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -2,6 +2,7 @@ package morph import ( "encoding/hex" + "fmt" "os" "path/filepath" "strconv" @@ -37,13 +38,22 @@ func TestInitialize(t *testing.T) { t.Run("7 nodes", func(t *testing.T) { testInitialize(t, 7) }) + t.Run("16 nodes", func(t *testing.T) { + testInitialize(t, 16) + }) + t.Run("max nodes", func(t *testing.T) { + testInitialize(t, maxAlphabetNodes) + }) + t.Run("too many nodes", func(t *testing.T) { + require.ErrorIs(t, generateTestData(t, t.TempDir(), maxAlphabetNodes+1), ErrTooManyAlphabetNodes) + }) } func testInitialize(t *testing.T, committeeSize int) { testdataDir := t.TempDir() v := viper.GetViper() - generateTestData(t, testdataDir, committeeSize) + require.NoError(t, generateTestData(t, testdataDir, committeeSize)) v.Set(protoConfigPath, filepath.Join(testdataDir, protoFileName)) // Set to the path or remove the next statement to download from the network. @@ -74,25 +84,33 @@ func testInitialize(t *testing.T, committeeSize int) { }) } -func generateTestData(t *testing.T, dir string, size int) { +func generateTestData(t *testing.T, dir string, size int) error { v := viper.GetViper() v.Set(alphabetWalletsFlag, dir) sizeStr := strconv.FormatUint(uint64(size), 10) - require.NoError(t, generateAlphabetCmd.Flags().Set(alphabetSizeFlag, sizeStr)) + if err := generateAlphabetCmd.Flags().Set(alphabetSizeFlag, sizeStr); err != nil { + return err + } setTestCredentials(v, size) - require.NoError(t, generateAlphabetCreds(generateAlphabetCmd, nil)) + if err := generateAlphabetCreds(generateAlphabetCmd, nil); err != nil { + return err + } var pubs []string for i := 0; i < size; i++ { p := filepath.Join(dir, innerring.GlagoliticLetter(i).String()+".json") w, err := wallet.NewWalletFromFile(p) - require.NoError(t, err, "wallet doesn't exist") + if err != nil { + return fmt.Errorf("wallet doesn't exist: %w", err) + } for _, acc := range w.Accounts { if acc.Label == singleAccountName { pub, ok := vm.ParseSignatureContract(acc.Contract.Script) - require.True(t, ok) + if !ok { + return fmt.Errorf("could not parse signature script for %s", acc.Address) + } pubs = append(pubs, hex.EncodeToString(pub)) continue } @@ -101,16 +119,18 @@ func generateTestData(t *testing.T, dir string, size int) { cfg := config.Config{} cfg.ProtocolConfiguration.Magic = 12345 - cfg.ProtocolConfiguration.ValidatorsCount = size + cfg.ProtocolConfiguration.ValidatorsCount = uint32(size) cfg.ProtocolConfiguration.TimePerBlock = time.Second cfg.ProtocolConfiguration.StandbyCommittee = pubs // sorted by glagolic letters cfg.ProtocolConfiguration.P2PSigExtensions = true cfg.ProtocolConfiguration.VerifyTransactions = true data, err := yaml.Marshal(cfg) - require.NoError(t, err) + if err != nil { + return err + } protoPath := filepath.Join(dir, protoFileName) - require.NoError(t, os.WriteFile(protoPath, data, os.ModePerm)) + return os.WriteFile(protoPath, data, os.ModePerm) } func setTestCredentials(v *viper.Viper, size int) { diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index 816f9da4c..0367f7479 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -51,7 +51,7 @@ type localClient struct { maxGasInvoke int64 } -func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (*localClient, error) { +func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet, dumpPath string) (*localClient, error) { cfg, err := config.LoadFile(v.GetString(protoConfigPath)) if err != nil { return nil, err @@ -62,7 +62,7 @@ func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet return nil, err } - m := smartcontract.GetDefaultHonestNodeCount(cfg.ProtocolConfiguration.ValidatorsCount) + m := smartcontract.GetDefaultHonestNodeCount(int(cfg.ProtocolConfiguration.ValidatorsCount)) accounts := make([]*wallet.Account, len(wallets)) for i := range accounts { accounts[i], err = getWalletAccount(wallets[i], consensusAccountName) @@ -87,7 +87,6 @@ func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet go bc.Run() - dumpPath := v.GetString(localDumpFlag) if cmd.Name() != "init" { f, err := os.OpenFile(dumpPath, os.O_RDONLY, 0600) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 431be125c..6a9e5b9c1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -77,7 +77,6 @@ var ( _ = viper.BindPFlag(containerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) _ = viper.BindPFlag(withdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) _ = viper.BindPFlag(protoConfigPath, cmd.Flags().Lookup(protoConfigPath)) - _ = viper.BindPFlag(localDumpFlag, cmd.Flags().Lookup(localDumpFlag)) }, RunE: initializeSideChainCmd, } diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index cbf19eb4b..babc0ef38 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -13,7 +13,7 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" ) @@ -37,8 +37,8 @@ func (x BalanceOfRes) Balance() accounting.Decimal { // BalanceOf requests the current balance of a FrostFS user. // // Returns any error which prevented the operation from completing correctly in error return. -func BalanceOf(prm BalanceOfPrm) (res BalanceOfRes, err error) { - res.cliRes, err = prm.cli.BalanceGet(context.Background(), prm.PrmBalanceGet) +func BalanceOf(ctx context.Context, prm BalanceOfPrm) (res BalanceOfRes, err error) { + res.cliRes, err = prm.cli.BalanceGet(ctx, prm.PrmBalanceGet) return } @@ -62,8 +62,8 @@ func (x ListContainersRes) IDList() []cid.ID { // ListContainers requests a list of FrostFS user's containers. // // Returns any error which prevented the operation from completing correctly in error return. -func ListContainers(prm ListContainersPrm) (res ListContainersRes, err error) { - res.cliRes, err = prm.cli.ContainerList(context.Background(), prm.PrmContainerList) +func ListContainers(ctx context.Context, prm ListContainersPrm) (res ListContainersRes, err error) { + res.cliRes, err = prm.cli.ContainerList(ctx, prm.PrmContainerList) return } @@ -92,8 +92,8 @@ func (x PutContainerRes) ID() cid.ID { // Success can be verified by reading by identifier. // // Returns any error which prevented the operation from completing correctly in error return. -func PutContainer(prm PutContainerPrm) (res PutContainerRes, err error) { - cliRes, err := prm.cli.ContainerPut(context.Background(), prm.PrmContainerPut) +func PutContainer(ctx context.Context, prm PutContainerPrm) (res PutContainerRes, err error) { + cliRes, err := prm.cli.ContainerPut(ctx, prm.PrmContainerPut) if err == nil { res.cnr = cliRes.ID() } @@ -125,20 +125,20 @@ func (x GetContainerRes) Container() containerSDK.Container { // GetContainer reads a container from FrostFS by ID. // // Returns any error which prevented the operation from completing correctly in error return. -func GetContainer(prm GetContainerPrm) (res GetContainerRes, err error) { - res.cliRes, err = prm.cli.ContainerGet(context.Background(), prm.cliPrm) +func GetContainer(ctx context.Context, prm GetContainerPrm) (res GetContainerRes, err error) { + res.cliRes, err = prm.cli.ContainerGet(ctx, prm.cliPrm) return } // IsACLExtendable checks if ACL of the container referenced by the given identifier // can be extended. Client connection MUST BE correctly established in advance. -func IsACLExtendable(c *client.Client, cnr cid.ID) (bool, error) { +func IsACLExtendable(ctx context.Context, c *client.Client, cnr cid.ID) (bool, error) { var prm GetContainerPrm prm.SetClient(c) prm.SetContainer(cnr) - res, err := GetContainer(prm) + res, err := GetContainer(ctx, prm) if err != nil { return false, fmt.Errorf("get container from the FrostFS: %w", err) } @@ -163,8 +163,8 @@ type DeleteContainerRes struct{} // Success can be verified by reading by identifier. // // Returns any error which prevented the operation from completing correctly in error return. -func DeleteContainer(prm DeleteContainerPrm) (res DeleteContainerRes, err error) { - _, err = prm.cli.ContainerDelete(context.Background(), prm.PrmContainerDelete) +func DeleteContainer(ctx context.Context, prm DeleteContainerPrm) (res DeleteContainerRes, err error) { + _, err = prm.cli.ContainerDelete(ctx, prm.PrmContainerDelete) return } @@ -188,8 +188,8 @@ func (x EACLRes) EACL() eacl.Table { // EACL reads eACL table from FrostFS by container ID. // // Returns any error which prevented the operation from completing correctly in error return. -func EACL(prm EACLPrm) (res EACLRes, err error) { - res.cliRes, err = prm.cli.ContainerEACL(context.Background(), prm.PrmContainerEACL) +func EACL(ctx context.Context, prm EACLPrm) (res EACLRes, err error) { + res.cliRes, err = prm.cli.ContainerEACL(ctx, prm.PrmContainerEACL) return } @@ -211,8 +211,8 @@ type SetEACLRes struct{} // Success can be verified by reading by container identifier. // // Returns any error which prevented the operation from completing correctly in error return. -func SetEACL(prm SetEACLPrm) (res SetEACLRes, err error) { - _, err = prm.cli.ContainerSetEACL(context.Background(), prm.PrmContainerSetEACL) +func SetEACL(ctx context.Context, prm SetEACLPrm) (res SetEACLRes, err error) { + _, err = prm.cli.ContainerSetEACL(ctx, prm.PrmContainerSetEACL) return } @@ -236,8 +236,8 @@ func (x NetworkInfoRes) NetworkInfo() netmap.NetworkInfo { // NetworkInfo reads information about the FrostFS network. // // Returns any error which prevented the operation from completing correctly in error return. -func NetworkInfo(prm NetworkInfoPrm) (res NetworkInfoRes, err error) { - res.cliRes, err = prm.cli.NetworkInfo(context.Background(), prm.PrmNetworkInfo) +func NetworkInfo(ctx context.Context, prm NetworkInfoPrm) (res NetworkInfoRes, err error) { + res.cliRes, err = prm.cli.NetworkInfo(ctx, prm.PrmNetworkInfo) return } @@ -266,8 +266,8 @@ func (x NodeInfoRes) LatestVersion() version.Version { // NodeInfo requests information about the remote server from FrostFS netmap. // // Returns any error which prevented the operation from completing correctly in error return. -func NodeInfo(prm NodeInfoPrm) (res NodeInfoRes, err error) { - res.cliRes, err = prm.cli.EndpointInfo(context.Background(), prm.PrmEndpointInfo) +func NodeInfo(ctx context.Context, prm NodeInfoPrm) (res NodeInfoRes, err error) { + res.cliRes, err = prm.cli.EndpointInfo(ctx, prm.PrmEndpointInfo) return } @@ -290,8 +290,8 @@ func (x NetMapSnapshotRes) NetMap() netmap.NetMap { // NetMapSnapshot requests current network view of the remote server. // // Returns any error which prevented the operation from completing correctly in error return. -func NetMapSnapshot(prm NetMapSnapshotPrm) (res NetMapSnapshotRes, err error) { - res.cliRes, err = prm.cli.NetMapSnapshot(context.Background(), client.PrmNetMapSnapshot{}) +func NetMapSnapshot(ctx context.Context, prm NetMapSnapshotPrm) (res NetMapSnapshotRes, err error) { + res.cliRes, err = prm.cli.NetMapSnapshot(ctx, client.PrmNetMapSnapshot{}) return } @@ -319,8 +319,8 @@ func (x CreateSessionRes) SessionKey() []byte { // CreateSession opens a new unlimited session with the remote node. // // Returns any error which prevented the operation from completing correctly in error return. -func CreateSession(prm CreateSessionPrm) (res CreateSessionRes, err error) { - res.cliRes, err = prm.cli.SessionCreate(context.Background(), prm.PrmSessionCreate) +func CreateSession(ctx context.Context, prm CreateSessionPrm) (res CreateSessionRes, err error) { + res.cliRes, err = prm.cli.SessionCreate(ctx, prm.PrmSessionCreate) return } @@ -329,15 +329,19 @@ func CreateSession(prm CreateSessionPrm) (res CreateSessionRes, err error) { type PutObjectPrm struct { commonObjectPrm - hdr *object.Object + copyNum []uint32 + + hdr *objectSDK.Object rdr io.Reader - headerCallback func(*object.Object) + headerCallback func(*objectSDK.Object) + + prepareLocally bool } // SetHeader sets object header. -func (x *PutObjectPrm) SetHeader(hdr *object.Object) { +func (x *PutObjectPrm) SetHeader(hdr *objectSDK.Object) { x.hdr = hdr } @@ -348,10 +352,51 @@ func (x *PutObjectPrm) SetPayloadReader(rdr io.Reader) { // SetHeaderCallback sets callback which is called on the object after the header is received // but before the payload is written. -func (x *PutObjectPrm) SetHeaderCallback(f func(*object.Object)) { +func (x *PutObjectPrm) SetHeaderCallback(f func(*objectSDK.Object)) { x.headerCallback = f } +// SetCopiesNumberByVectors sets ordered list of minimal required object copies numbers +// per placement vector. +func (x *PutObjectPrm) SetCopiesNumberByVectors(copiesNumbers []uint32) { + x.copyNum = copiesNumbers +} + +// PrepareLocally generate object header on the client side. +// For big object - split locally too. +func (x *PutObjectPrm) PrepareLocally() { + x.prepareLocally = true +} + +func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) { + var putPrm client.PrmObjectPutInit + if !x.prepareLocally && x.sessionToken != nil { + putPrm.WithinSession(*x.sessionToken) + } + + if x.bearerToken != nil { + putPrm.WithBearerToken(*x.bearerToken) + } + + if x.local { + putPrm.MarkLocal() + } + + putPrm.WithXHeaders(x.xHeaders...) + putPrm.SetCopiesNumberByVectors(x.copyNum) + + if x.prepareLocally { + res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{}) + if err != nil { + return client.PrmObjectPutInit{}, err + } + putPrm.WithObjectMaxSize(res.Info().MaxObjectSize()) + putPrm.WithEpochSource(epochSource(res.Info().CurrentEpoch())) + putPrm.WithoutHomomorphicHash(res.Info().HomomorphicHashingDisabled()) + } + return putPrm, nil +} + // PutObjectRes groups the resulting values of PutObject operation. type PutObjectRes struct { id oid.ID @@ -362,32 +407,26 @@ func (x PutObjectRes) ID() oid.ID { return x.id } +type epochSource uint64 + +func (s epochSource) CurrentEpoch() uint64 { + return uint64(s) +} + // PutObject saves the object in FrostFS network. // // Returns any error which prevented the operation from completing correctly in error return. -func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { - var putPrm client.PrmObjectPutInit - - if prm.sessionToken != nil { - putPrm.WithinSession(*prm.sessionToken) +func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { + sdkPrm, err := prm.convertToSDKPrm(ctx) + if err != nil { + return nil, fmt.Errorf("unable to create parameters of object put operation: %w", err) } - - if prm.bearerToken != nil { - putPrm.WithBearerToken(*prm.bearerToken) - } - - if prm.local { - putPrm.MarkLocal() - } - - putPrm.WithXHeaders(prm.xHeaders...) - - wrt, err := prm.cli.ObjectPutInit(context.Background(), putPrm) + wrt, err := prm.cli.ObjectPutInit(ctx, sdkPrm) if err != nil { return nil, fmt.Errorf("init object writing: %w", err) } - if wrt.WriteHeader(*prm.hdr) { + if wrt.WriteHeader(ctx, *prm.hdr) { if prm.headerCallback != nil { prm.headerCallback(prm.hdr) } @@ -417,7 +456,7 @@ func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { for { n, err = prm.rdr.Read(buf) if n > 0 { - if !wrt.WritePayloadChunk(buf[:n]) { + if !wrt.WritePayloadChunk(ctx, buf[:n]) { break } @@ -433,7 +472,7 @@ func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { } } - cliRes, err := wrt.Close() + cliRes, err := wrt.Close(ctx) if err != nil { // here err already carries both status and client errors return nil, fmt.Errorf("client failure: %w", err) } @@ -462,7 +501,7 @@ func (x DeleteObjectRes) Tombstone() oid.ID { // DeleteObject marks an object to be removed from FrostFS through tombstone placement. // // Returns any error which prevented the operation from completing correctly in error return. -func DeleteObject(prm DeleteObjectPrm) (*DeleteObjectRes, error) { +func DeleteObject(ctx context.Context, prm DeleteObjectPrm) (*DeleteObjectRes, error) { var delPrm client.PrmObjectDelete delPrm.FromContainer(prm.objAddr.Container()) delPrm.ByID(prm.objAddr.Object()) @@ -477,7 +516,7 @@ func DeleteObject(prm DeleteObjectPrm) (*DeleteObjectRes, error) { delPrm.WithXHeaders(prm.xHeaders...) - cliRes, err := prm.cli.ObjectDelete(context.Background(), delPrm) + cliRes, err := prm.cli.ObjectDelete(ctx, delPrm) if err != nil { return nil, fmt.Errorf("remove object via client: %w", err) } @@ -493,22 +532,22 @@ type GetObjectPrm struct { objectAddressPrm rawPrm payloadWriterPrm - headerCallback func(*object.Object) + headerCallback func(*objectSDK.Object) } // SetHeaderCallback sets callback which is called on the object after the header is received // but before the payload is written. -func (p *GetObjectPrm) SetHeaderCallback(f func(*object.Object)) { +func (p *GetObjectPrm) SetHeaderCallback(f func(*objectSDK.Object)) { p.headerCallback = f } // GetObjectRes groups the resulting values of GetObject operation. type GetObjectRes struct { - hdr *object.Object + hdr *objectSDK.Object } // Header returns the header of the request object. -func (x GetObjectRes) Header() *object.Object { +func (x GetObjectRes) Header() *objectSDK.Object { return x.hdr } @@ -518,7 +557,7 @@ func (x GetObjectRes) Header() *object.Object { // // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. -func GetObject(prm GetObjectPrm) (*GetObjectRes, error) { +func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { var getPrm client.PrmObjectGet getPrm.FromContainer(prm.objAddr.Container()) getPrm.ByID(prm.objAddr.Object()) @@ -541,12 +580,12 @@ func GetObject(prm GetObjectPrm) (*GetObjectRes, error) { getPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectGetInit(context.Background(), getPrm) + rdr, err := prm.cli.ObjectGetInit(ctx, getPrm) if err != nil { return nil, fmt.Errorf("init object reading on client: %w", err) } - var hdr object.Object + var hdr objectSDK.Object if !rdr.ReadHeader(&hdr) { _, err = rdr.Close() @@ -582,11 +621,11 @@ func (x *HeadObjectPrm) SetMainOnlyFlag(v bool) { // HeadObjectRes groups the resulting values of HeadObject operation. type HeadObjectRes struct { - hdr *object.Object + hdr *objectSDK.Object } // Header returns the requested object header. -func (x HeadObjectRes) Header() *object.Object { +func (x HeadObjectRes) Header() *objectSDK.Object { return x.hdr } @@ -594,7 +633,7 @@ func (x HeadObjectRes) Header() *object.Object { // // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. -func HeadObject(prm HeadObjectPrm) (*HeadObjectRes, error) { +func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) { var cliPrm client.PrmObjectHead cliPrm.FromContainer(prm.objAddr.Container()) cliPrm.ByID(prm.objAddr.Object()) @@ -617,12 +656,12 @@ func HeadObject(prm HeadObjectPrm) (*HeadObjectRes, error) { cliPrm.WithXHeaders(prm.xHeaders...) - res, err := prm.cli.ObjectHead(context.Background(), cliPrm) + res, err := prm.cli.ObjectHead(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("read object header via client: %w", err) } - var hdr object.Object + var hdr objectSDK.Object if !res.ReadHeader(&hdr) { return nil, fmt.Errorf("missing header in response") @@ -638,11 +677,11 @@ type SearchObjectsPrm struct { commonObjectPrm containerIDPrm - filters object.SearchFilters + filters objectSDK.SearchFilters } // SetFilters sets search filters. -func (x *SearchObjectsPrm) SetFilters(filters object.SearchFilters) { +func (x *SearchObjectsPrm) SetFilters(filters objectSDK.SearchFilters) { x.filters = filters } @@ -659,7 +698,7 @@ func (x SearchObjectsRes) IDList() []oid.ID { // SearchObjects selects objects from the container which match the filters. // // Returns any error which prevented the operation from completing correctly in error return. -func SearchObjects(prm SearchObjectsPrm) (*SearchObjectsRes, error) { +func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes, error) { var cliPrm client.PrmObjectSearch cliPrm.InContainer(prm.cnrID) cliPrm.SetFilters(prm.filters) @@ -678,7 +717,7 @@ func SearchObjects(prm SearchObjectsPrm) (*SearchObjectsRes, error) { cliPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectSearchInit(context.Background(), cliPrm) + rdr, err := prm.cli.ObjectSearchInit(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("init object search: %w", err) } @@ -715,7 +754,7 @@ type HashPayloadRangesPrm struct { tz bool - rngs []*object.Range + rngs []*objectSDK.Range salt []byte } @@ -726,7 +765,7 @@ func (x *HashPayloadRangesPrm) TZ() { } // SetRanges sets a list of payload ranges to hash. -func (x *HashPayloadRangesPrm) SetRanges(rngs []*object.Range) { +func (x *HashPayloadRangesPrm) SetRanges(rngs []*objectSDK.Range) { x.rngs = rngs } @@ -749,7 +788,7 @@ func (x HashPayloadRangesRes) HashList() [][]byte { // // Returns any error which prevented the operation from completing correctly in error return. // Returns an error if number of received hashes differs with the number of requested ranges. -func HashPayloadRanges(prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) { +func HashPayloadRanges(ctx context.Context, prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) { var cliPrm client.PrmObjectHash cliPrm.FromContainer(prm.objAddr.Container()) cliPrm.ByID(prm.objAddr.Object()) @@ -783,7 +822,7 @@ func HashPayloadRanges(prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) cliPrm.WithXHeaders(prm.xHeaders...) - res, err := prm.cli.ObjectHash(context.Background(), cliPrm) + res, err := prm.cli.ObjectHash(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("read payload hashes via client: %w", err) } @@ -800,11 +839,11 @@ type PayloadRangePrm struct { rawPrm payloadWriterPrm - rng *object.Range + rng *objectSDK.Range } // SetRange sets payload range to read. -func (x *PayloadRangePrm) SetRange(rng *object.Range) { +func (x *PayloadRangePrm) SetRange(rng *objectSDK.Range) { x.rng = rng } @@ -817,7 +856,7 @@ type PayloadRangeRes struct{} // // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. -func PayloadRange(prm PayloadRangePrm) (*PayloadRangeRes, error) { +func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, error) { var cliPrm client.PrmObjectRange cliPrm.FromContainer(prm.objAddr.Container()) cliPrm.ByID(prm.objAddr.Object()) @@ -843,7 +882,7 @@ func PayloadRange(prm PayloadRangePrm) (*PayloadRangeRes, error) { cliPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectRangeInit(context.Background(), cliPrm) + rdr, err := prm.cli.ObjectRangeInit(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("init payload reading: %w", err) } @@ -877,12 +916,12 @@ type SyncContainerRes struct{} // Interrupts on any writer error. // // Panics if a container passed as a parameter is nil. -func SyncContainerSettings(prm SyncContainerPrm) (*SyncContainerRes, error) { +func SyncContainerSettings(ctx context.Context, prm SyncContainerPrm) (*SyncContainerRes, error) { if prm.c == nil { panic("sync container settings with the network: nil container") } - err := client.SyncContainerWithNetwork(context.Background(), prm.c, prm.cli) + err := client.SyncContainerWithNetwork(ctx, prm.c, prm.cli) if err != nil { return nil, err } diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index 13dacc04c..79d3dcb0d 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -12,9 +12,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" "github.com/spf13/viper" + "google.golang.org/grpc" ) var errInvalidEndpoint = errors.New("provided RPC endpoint is incorrect") @@ -59,6 +61,9 @@ func GetSDKClient(ctx context.Context, cmd *cobra.Command, key *ecdsa.PrivateKey common.PrintVerbose(cmd, "Set request timeout to %s.", timeout) } + prmDial.SetGRPCDialOptions( + grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor()), + grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor())) c.Init(prmInit) diff --git a/cmd/frostfs-cli/internal/common/tracing.go b/cmd/frostfs-cli/internal/common/tracing.go new file mode 100644 index 000000000..30c2f2b1a --- /dev/null +++ b/cmd/frostfs-cli/internal/common/tracing.go @@ -0,0 +1,62 @@ +package common + +import ( + "context" + "sort" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/misc" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "github.com/spf13/cobra" + "go.opentelemetry.io/otel/trace" +) + +type spanKey struct{} + +// StopClientCommandSpan stops tracing span for the command and prints trace ID on the standard output. +func StopClientCommandSpan(cmd *cobra.Command, _ []string) { + span, ok := cmd.Context().Value(spanKey{}).(trace.Span) + if !ok { + return + } + + span.End() + + // Noop provider cannot fail on flush. + _ = tracing.Shutdown(cmd.Context()) + + cmd.PrintErrf("Trace ID: %s\n", span.SpanContext().TraceID()) +} + +// StartClientCommandSpan starts tracing span for the command. +func StartClientCommandSpan(cmd *cobra.Command) { + enableTracing, err := cmd.Flags().GetBool(commonflags.TracingFlag) + if err != nil || !enableTracing { + return + } + + _, err = tracing.Setup(cmd.Context(), tracing.Config{ + Enabled: true, + Exporter: tracing.NoOpExporter, + Service: "frostfs-cli", + Version: misc.Version, + }) + commonCmd.ExitOnErr(cmd, "init tracing: %w", err) + + var components sort.StringSlice + for c := cmd; c != nil; c = c.Parent() { + components = append(components, c.Name()) + } + for i, j := 0, len(components)-1; i < j; { + components.Swap(i, j) + i++ + j-- + } + + operation := strings.Join(components, ".") + ctx, span := tracing.StartSpanFromContext(cmd.Context(), operation) + ctx = context.WithValue(ctx, spanKey{}, span) + cmd.SetContext(ctx) +} diff --git a/cmd/frostfs-cli/internal/commonflags/flags.go b/cmd/frostfs-cli/internal/commonflags/flags.go index 810e62107..5049dc3b1 100644 --- a/cmd/frostfs-cli/internal/commonflags/flags.go +++ b/cmd/frostfs-cli/internal/commonflags/flags.go @@ -47,6 +47,9 @@ const ( OIDFlag = "oid" OIDFlagUsage = "Object ID." + + TracingFlag = "trace" + TracingFlagUsage = "Generate trace ID and print it." ) // Init adds common flags to the command: @@ -54,12 +57,14 @@ const ( // - WalletPath, // - Account, // - RPC, +// - Tracing, // - Timeout. func Init(cmd *cobra.Command) { InitWithoutRPC(cmd) ff := cmd.Flags() ff.StringP(RPC, RPCShorthand, RPCDefault, RPCUsage) + ff.Bool(TracingFlag, false, TracingFlagUsage) ff.DurationP(Timeout, TimeoutShorthand, TimeoutDefault, TimeoutUsage) } diff --git a/cmd/frostfs-cli/modules/accounting/balance.go b/cmd/frostfs-cli/modules/accounting/balance.go index bec40f1ff..5ed8f9403 100644 --- a/cmd/frostfs-cli/modules/accounting/balance.go +++ b/cmd/frostfs-cli/modules/accounting/balance.go @@ -41,7 +41,7 @@ var accountingBalanceCmd = &cobra.Command{ prm.SetClient(cli) prm.SetAccount(idUser) - res, err := internalclient.BalanceOf(prm) + res, err := internalclient.BalanceOf(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) // print to stdout diff --git a/cmd/frostfs-cli/modules/accounting/root.go b/cmd/frostfs-cli/modules/accounting/root.go index 8ab8aa125..f94488b6f 100644 --- a/cmd/frostfs-cli/modules/accounting/root.go +++ b/cmd/frostfs-cli/modules/accounting/root.go @@ -1,6 +1,7 @@ package accounting import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -17,7 +18,9 @@ var Cmd = &cobra.Command{ _ = viper.BindPFlag(commonflags.WalletPath, flags.Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, flags.Lookup(commonflags.Account)) _ = viper.BindPFlag(commonflags.RPC, flags.Lookup(commonflags.RPC)) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/bearer/create.go b/cmd/frostfs-cli/modules/bearer/create.go index 2f1623d9b..b85115047 100644 --- a/cmd/frostfs-cli/modules/bearer/create.go +++ b/cmd/frostfs-cli/modules/bearer/create.go @@ -24,6 +24,7 @@ const ( ownerFlag = "owner" outFlag = "out" jsonFlag = commonflags.JSON + impersonateFlag = "impersonate" ) var createCmd = &cobra.Command{ @@ -39,19 +40,20 @@ is set to current epoch + n. } func init() { - createCmd.Flags().StringP(eaclFlag, "e", "", "Path to the extended ACL table") - createCmd.Flags().StringP(issuedAtFlag, "i", "", "Epoch to issue token at") - createCmd.Flags().StringP(notValidBeforeFlag, "n", "", "Not valid before epoch") + createCmd.Flags().StringP(eaclFlag, "e", "", "Path to the extended ACL table (mutually exclusive with --impersonate flag)") + createCmd.Flags().StringP(issuedAtFlag, "i", "+0", "Epoch to issue token at") + createCmd.Flags().StringP(notValidBeforeFlag, "n", "+0", "Not valid before epoch") createCmd.Flags().StringP(commonflags.ExpireAt, "x", "", "The last active epoch for the token") createCmd.Flags().StringP(ownerFlag, "o", "", "Token owner") createCmd.Flags().String(outFlag, "", "File to write token to") createCmd.Flags().Bool(jsonFlag, false, "Output token in JSON") + createCmd.Flags().Bool(impersonateFlag, false, "Mark token as impersonate to consider the token signer as the request owner (mutually exclusive with --eacl flag)") createCmd.Flags().StringP(commonflags.RPC, commonflags.RPCShorthand, commonflags.RPCDefault, commonflags.RPCUsage) + createCmd.MarkFlagsMutuallyExclusive(eaclFlag, impersonateFlag) + _ = cobra.MarkFlagFilename(createCmd.Flags(), eaclFlag) - _ = cobra.MarkFlagRequired(createCmd.Flags(), issuedAtFlag) - _ = cobra.MarkFlagRequired(createCmd.Flags(), notValidBeforeFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), commonflags.ExpireAt) _ = cobra.MarkFlagRequired(createCmd.Flags(), ownerFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), outFlag) @@ -68,10 +70,14 @@ func createToken(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "can't parse --"+notValidBeforeFlag+" flag: %w", err) if iatRelative || expRelative || nvbRelative { + endpoint, _ := cmd.Flags().GetString(commonflags.RPC) + if len(endpoint) == 0 { + commonCmd.ExitOnErr(cmd, "can't fetch current epoch: %w", fmt.Errorf("'%s' flag value must be specified", commonflags.RPC)) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - endpoint, _ := cmd.Flags().GetString(commonflags.RPC) currEpoch, err := internalclient.GetCurrentEpoch(ctx, cmd, endpoint) commonCmd.ExitOnErr(cmd, "can't fetch current epoch: %w", err) @@ -101,6 +107,9 @@ func createToken(cmd *cobra.Command, _ []string) { b.SetIat(iat) b.ForUser(ownerID) + impersonate, _ := cmd.Flags().GetBool(impersonateFlag) + b.SetImpersonate(impersonate) + eaclPath, _ := cmd.Flags().GetString(eaclFlag) if eaclPath != "" { table := eaclSDK.NewTable() diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 873ef3235..e5bbeacd4 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -48,7 +48,7 @@ It will be stored in sidechain when inner ring will accepts it.`, var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - resmap, err := internalclient.NetMapSnapshot(prm) + resmap, err := internalclient.NetMapSnapshot(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "unable to get netmap snapshot to validate container placement, "+ "use --force option to skip this check: %w", err) @@ -96,7 +96,7 @@ It will be stored in sidechain when inner ring will accepts it.`, syncContainerPrm.SetClient(cli) syncContainerPrm.SetContainer(&cnr) - _, err = internalclient.SyncContainerSettings(syncContainerPrm) + _, err = internalclient.SyncContainerSettings(cmd.Context(), syncContainerPrm) commonCmd.ExitOnErr(cmd, "syncing container's settings rpc error: %w", err) var putPrm internalclient.PutContainerPrm @@ -107,7 +107,7 @@ It will be stored in sidechain when inner ring will accepts it.`, putPrm.WithinSession(*tok) } - res, err := internalclient.PutContainer(putPrm) + res, err := internalclient.PutContainer(cmd.Context(), putPrm) commonCmd.ExitOnErr(cmd, "put container rpc error: %w", err) id := res.ID() @@ -124,7 +124,7 @@ It will be stored in sidechain when inner ring will accepts it.`, for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) - _, err := internalclient.GetContainer(getPrm) + _, err := internalclient.GetContainer(cmd.Context(), getPrm) if err == nil { cmd.Println("container has been persisted on sidechain") return @@ -141,6 +141,7 @@ func initContainerCreateCmd() { // Init common flags flags.StringP(commonflags.RPC, commonflags.RPCShorthand, commonflags.RPCDefault, commonflags.RPCUsage) + flags.Bool(commonflags.TracingFlag, false, commonflags.TracingFlagUsage) flags.DurationP(commonflags.Timeout, commonflags.TimeoutShorthand, commonflags.TimeoutDefault, commonflags.TimeoutUsage) flags.StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, commonflags.WalletPathDefault, commonflags.WalletPathUsage) flags.StringP(commonflags.Account, commonflags.AccountShorthand, commonflags.AccountDefault, commonflags.AccountUsage) diff --git a/cmd/frostfs-cli/modules/container/delete.go b/cmd/frostfs-cli/modules/container/delete.go index 308c7b942..f5b69edfd 100644 --- a/cmd/frostfs-cli/modules/container/delete.go +++ b/cmd/frostfs-cli/modules/container/delete.go @@ -34,7 +34,7 @@ Only owner of the container has a permission to remove container.`, getPrm.SetClient(cli) getPrm.SetContainer(id) - resGet, err := internalclient.GetContainer(getPrm) + resGet, err := internalclient.GetContainer(cmd.Context(), getPrm) commonCmd.ExitOnErr(cmd, "can't get the container: %w", err) owner := resGet.Container().Owner() @@ -72,7 +72,7 @@ Only owner of the container has a permission to remove container.`, common.PrintVerbose(cmd, "Searching for LOCK objects...") - res, err := internalclient.SearchObjects(searchPrm) + res, err := internalclient.SearchObjects(cmd.Context(), searchPrm) commonCmd.ExitOnErr(cmd, "can't search for LOCK objects: %w", err) if len(res.IDList()) != 0 { @@ -91,7 +91,7 @@ Only owner of the container has a permission to remove container.`, delPrm.WithinSession(*tok) } - _, err := internalclient.DeleteContainer(delPrm) + _, err := internalclient.DeleteContainer(cmd.Context(), delPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) cmd.Println("container delete method invoked") @@ -106,7 +106,7 @@ Only owner of the container has a permission to remove container.`, for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) - _, err := internalclient.GetContainer(getPrm) + _, err := internalclient.GetContainer(cmd.Context(), getPrm) if err != nil { cmd.Println("container has been removed:", containerID) return @@ -124,6 +124,7 @@ func initContainerDeleteCmd() { flags.StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, commonflags.WalletPathDefault, commonflags.WalletPathUsage) flags.StringP(commonflags.Account, commonflags.AccountShorthand, commonflags.AccountDefault, commonflags.AccountUsage) flags.StringP(commonflags.RPC, commonflags.RPCShorthand, commonflags.RPCDefault, commonflags.RPCUsage) + flags.Bool(commonflags.TracingFlag, false, commonflags.TracingFlagUsage) flags.StringVar(&containerID, commonflags.CIDFlag, "", commonflags.CIDFlagUsage) flags.BoolVar(&containerAwait, "await", false, "Block execution until container is removed") diff --git a/cmd/frostfs-cli/modules/container/get.go b/cmd/frostfs-cli/modules/container/get.go index 2db1f7c8d..90bcc190a 100644 --- a/cmd/frostfs-cli/modules/container/get.go +++ b/cmd/frostfs-cli/modules/container/get.go @@ -151,7 +151,7 @@ func getContainer(cmd *cobra.Command) (container.Container, *ecdsa.PrivateKey) { prm.SetClient(cli) prm.SetContainer(id) - res, err := internalclient.GetContainer(prm) + res, err := internalclient.GetContainer(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) cnr = res.Container() diff --git a/cmd/frostfs-cli/modules/container/get_eacl.go b/cmd/frostfs-cli/modules/container/get_eacl.go index bc04a84fc..21ea5b5bc 100644 --- a/cmd/frostfs-cli/modules/container/get_eacl.go +++ b/cmd/frostfs-cli/modules/container/get_eacl.go @@ -24,7 +24,7 @@ var getExtendedACLCmd = &cobra.Command{ eaclPrm.SetClient(cli) eaclPrm.SetContainer(id) - res, err := internalclient.EACL(eaclPrm) + res, err := internalclient.EACL(cmd.Context(), eaclPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) eaclTable := res.EACL() diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 33dd17943..189cc05c7 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -49,7 +49,7 @@ var listContainersCmd = &cobra.Command{ prm.SetClient(cli) prm.SetAccount(idUser) - res, err := internalclient.ListContainers(prm) + res, err := internalclient.ListContainers(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) var prmGet internalclient.GetContainerPrm @@ -63,7 +63,7 @@ var listContainersCmd = &cobra.Command{ } prmGet.SetContainer(cnrID) - res, err := internalclient.GetContainer(prmGet) + res, err := internalclient.GetContainer(cmd.Context(), prmGet) if err != nil { cmd.Printf(" failed to read attributes: %v\n", err) continue @@ -78,7 +78,8 @@ var listContainersCmd = &cobra.Command{ if flagVarListPrintAttr { cnr.IterateAttributes(func(key, val string) { if !strings.HasPrefix(key, container.SysAttributePrefix) && !strings.HasPrefix(key, container.SysAttributePrefixNeoFS) { - // FIXME(@cthulhu-rider): neofs-sdk-go#314 use dedicated method to skip system attributes + // FIXME(@cthulhu-rider): https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/97 + // Use dedicated method to skip system attributes. cmd.Printf(" %s: %s\n", key, val) } }) diff --git a/cmd/frostfs-cli/modules/container/list_objects.go b/cmd/frostfs-cli/modules/container/list_objects.go index aef4a1f80..1e0aeb4db 100644 --- a/cmd/frostfs-cli/modules/container/list_objects.go +++ b/cmd/frostfs-cli/modules/container/list_objects.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" objectCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/object" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -31,7 +31,7 @@ var listContainerObjectsCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { id := parseContainerID(cmd) - filters := new(object.SearchFilters) + filters := new(objectSDK.SearchFilters) filters.AddRootFilter() // search only user created objects cli := internalclient.GetSDKClientByFlag(cmd, key.GetOrGenerate(cmd), commonflags.RPC) @@ -51,7 +51,7 @@ var listContainerObjectsCmd = &cobra.Command{ prmSearch.SetContainerID(id) prmSearch.SetFilters(*filters) - res, err := internalclient.SearchObjects(prmSearch) + res, err := internalclient.SearchObjects(cmd.Context(), prmSearch) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) objectIDs := res.IDList() @@ -65,13 +65,14 @@ var listContainerObjectsCmd = &cobra.Command{ addr.SetObject(objectIDs[i]) prmHead.SetAddress(addr) - resHead, err := internalclient.HeadObject(prmHead) + resHead, err := internalclient.HeadObject(cmd.Context(), prmHead) if err == nil { attrs := resHead.Header().Attributes() for i := range attrs { attrKey := attrs[i].Key() if !strings.HasPrefix(attrKey, v2object.SysAttributePrefix) && !strings.HasPrefix(attrKey, v2object.SysAttributePrefixNeoFS) { - // FIXME(@cthulhu-rider): neofs-sdk-go#226 use dedicated method to skip system attributes + // FIXME(@cthulhu-rider): https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/97 + // Use dedicated method to skip system attributes. cmd.Printf(" %s: %s\n", attrKey, attrs[i].Value()) } } diff --git a/cmd/frostfs-cli/modules/container/nodes.go b/cmd/frostfs-cli/modules/container/nodes.go index d89772fcc..8b0f266a7 100644 --- a/cmd/frostfs-cli/modules/container/nodes.go +++ b/cmd/frostfs-cli/modules/container/nodes.go @@ -31,7 +31,7 @@ var containerNodesCmd = &cobra.Command{ var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - resmap, err := internalclient.NetMapSnapshot(prm) + resmap, err := internalclient.NetMapSnapshot(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "unable to get netmap snapshot", err) var id cid.ID diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go new file mode 100644 index 000000000..1ac41f08c --- /dev/null +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -0,0 +1,233 @@ +package container + +import ( + "bufio" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type policyPlaygroundREPL struct { + cmd *cobra.Command + args []string + nodes map[string]netmap.NodeInfo +} + +func newPolicyPlaygroundREPL(cmd *cobra.Command, args []string) (*policyPlaygroundREPL, error) { + return &policyPlaygroundREPL{ + cmd: cmd, + args: args, + nodes: map[string]netmap.NodeInfo{}, + }, nil +} + +func (repl *policyPlaygroundREPL) handleLs(args []string) error { + if len(args) > 0 { + return fmt.Errorf("too many arguments for command 'ls': got %d, want 0", len(args)) + } + i := 1 + for id, node := range repl.nodes { + var attrs []string + node.IterateAttributes(func(k, v string) { + attrs = append(attrs, fmt.Sprintf("%s:%q", k, v)) + }) + fmt.Printf("\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) + i++ + } + return nil +} + +func (repl *policyPlaygroundREPL) handleAdd(args []string) error { + if len(args) == 0 { + return fmt.Errorf("too few arguments for command 'add': got %d, want >0", len(args)) + } + id := args[0] + key, err := hex.DecodeString(id) + if err != nil { + return fmt.Errorf("node id must be a hex string: got %q: %v", id, err) + } + node := repl.nodes[id] + node.SetPublicKey(key) + for _, attr := range args[1:] { + kv := strings.Split(attr, ":") + if len(kv) != 2 { + return fmt.Errorf("node attributes must be in the format 'KEY:VALUE': got %q", attr) + } + node.SetAttribute(kv[0], kv[1]) + } + repl.nodes[id] = node + return nil +} + +func (repl *policyPlaygroundREPL) handleLoad(args []string) error { + if len(args) != 1 { + return fmt.Errorf("too few arguments for command 'add': got %d, want 1", len(args)) + } + + jsonNetmap := map[string]map[string]string{} + + b, err := os.ReadFile(args[0]) + if err != nil { + return fmt.Errorf("reading netmap file %q: %v", args[0], err) + } + + if err := json.Unmarshal(b, &jsonNetmap); err != nil { + return fmt.Errorf("decoding json netmap: %v", err) + } + + repl.nodes = make(map[string]netmap.NodeInfo) + for id, attrs := range jsonNetmap { + key, err := hex.DecodeString(id) + if err != nil { + return fmt.Errorf("node id must be a hex string: got %q: %v", id, err) + } + + node := repl.nodes[id] + node.SetPublicKey(key) + for k, v := range attrs { + node.SetAttribute(k, v) + } + repl.nodes[id] = node + } + + return nil +} + +func (repl *policyPlaygroundREPL) handleRemove(args []string) error { + if len(args) == 0 { + return fmt.Errorf("too few arguments for command 'remove': got %d, want >0", len(args)) + } + id := args[0] + if _, exists := repl.nodes[id]; exists { + delete(repl.nodes, id) + return nil + } + return fmt.Errorf("node not found: id=%q", id) +} + +func (repl *policyPlaygroundREPL) handleEval(args []string) error { + policyStr := strings.TrimSpace(strings.Join(args, " ")) + var nodes [][]netmap.NodeInfo + nm := repl.netMap() + + if strings.HasPrefix(policyStr, "CBF") || strings.HasPrefix(policyStr, "SELECT") || strings.HasPrefix(policyStr, "FILTER") { + // Assume that the input is a partial SELECT-FILTER expression. + // Full inline policies always start with UNIQUE or REP keywords, + // or different prefixes when it's the case of an external file. + sfExpr, err := netmap.DecodeSelectFilterString(policyStr) + if err != nil { + return fmt.Errorf("parsing select-filter expression: %v", err) + } + nodes, err = nm.SelectFilterNodes(sfExpr) + if err != nil { + return fmt.Errorf("building select-filter nodes: %v", err) + } + } else { + // Assume that the input is a full policy or input file otherwise. + placementPolicy, err := parseContainerPolicy(repl.cmd, policyStr) + if err != nil { + return fmt.Errorf("parsing placement policy: %v", err) + } + nodes, err = nm.ContainerNodes(*placementPolicy, nil) + if err != nil { + return fmt.Errorf("building container nodes: %v", err) + } + } + for i, ns := range nodes { + var ids []string + for _, node := range ns { + ids = append(ids, hex.EncodeToString(node.PublicKey())) + } + fmt.Printf("\t%2d: %v\n", i+1, ids) + } + + return nil +} + +func (repl *policyPlaygroundREPL) netMap() netmap.NetMap { + var nm netmap.NetMap + var nodes []netmap.NodeInfo + for _, node := range repl.nodes { + nodes = append(nodes, node) + } + nm.SetNodes(nodes) + return nm +} + +func (repl *policyPlaygroundREPL) run() error { + if len(viper.GetString(commonflags.RPC)) > 0 { + key := key.GetOrGenerate(repl.cmd) + cli := internalclient.GetSDKClientByFlag(repl.cmd, key, commonflags.RPC) + + var prm internalclient.NetMapSnapshotPrm + prm.SetClient(cli) + + resp, err := internalclient.NetMapSnapshot(repl.cmd.Context(), prm) + commonCmd.ExitOnErr(repl.cmd, "unable to get netmap snapshot to populate initial netmap: %w", err) + + for _, node := range resp.NetMap().Nodes() { + id := hex.EncodeToString(node.PublicKey()) + repl.nodes[id] = node + } + } + + cmdHandlers := map[string]func([]string) error{ + "list": repl.handleLs, + "ls": repl.handleLs, + "add": repl.handleAdd, + "load": repl.handleLoad, + "remove": repl.handleRemove, + "rm": repl.handleRemove, + "eval": repl.handleEval, + } + for reader := bufio.NewReader(os.Stdin); ; { + fmt.Print("> ") + line, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + return nil + } + return fmt.Errorf("reading line: %v", err) + } + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + cmd := parts[0] + handler, exists := cmdHandlers[cmd] + if exists { + if err := handler(parts[1:]); err != nil { + fmt.Printf("error: %v\n", err) + } + } else { + fmt.Printf("error: unknown command %q\n", cmd) + } + } +} + +var policyPlaygroundCmd = &cobra.Command{ + Use: "policy-playground", + Short: "A REPL for testing placement policies", + Long: `A REPL for testing placement policies. +If a wallet and endpoint is provided, the initial netmap data will be loaded from the snapshot of the node. Otherwise, an empty playground is created.`, + Run: func(cmd *cobra.Command, args []string) { + repl, err := newPolicyPlaygroundREPL(cmd, args) + commonCmd.ExitOnErr(cmd, "could not create policy playground: %w", err) + commonCmd.ExitOnErr(cmd, "policy playground failed: %w", repl.run()) + }, +} + +func initContainerPolicyPlaygroundCmd() { + commonflags.Init(policyPlaygroundCmd) +} diff --git a/cmd/frostfs-cli/modules/container/root.go b/cmd/frostfs-cli/modules/container/root.go index 30a82954a..f3c3e0e3a 100644 --- a/cmd/frostfs-cli/modules/container/root.go +++ b/cmd/frostfs-cli/modules/container/root.go @@ -1,6 +1,7 @@ package container import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -15,7 +16,9 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { @@ -28,6 +31,7 @@ func init() { getExtendedACLCmd, setExtendedACLCmd, containerNodesCmd, + policyPlaygroundCmd, } Cmd.AddCommand(containerChildCommand...) @@ -40,6 +44,7 @@ func init() { initContainerGetEACLCmd() initContainerSetEACLCmd() initContainerNodesCmd() + initContainerPolicyPlaygroundCmd() for _, containerCommand := range containerChildCommand { commonflags.InitAPI(containerCommand) diff --git a/cmd/frostfs-cli/modules/container/set_eacl.go b/cmd/frostfs-cli/modules/container/set_eacl.go index 0b781589f..c88d5767b 100644 --- a/cmd/frostfs-cli/modules/container/set_eacl.go +++ b/cmd/frostfs-cli/modules/container/set_eacl.go @@ -38,7 +38,7 @@ Container ID in EACL table will be substituted with ID from the CLI.`, if !flagVarsSetEACL.noPreCheck { cmd.Println("Checking the ability to modify access rights in the container...") - extendable, err := internalclient.IsACLExtendable(cli, id) + extendable, err := internalclient.IsACLExtendable(cmd.Context(), cli, id) commonCmd.ExitOnErr(cmd, "Extensibility check failure: %w", err) if !extendable { @@ -56,7 +56,7 @@ Container ID in EACL table will be substituted with ID from the CLI.`, setEACLPrm.WithinSession(*tok) } - _, err := internalclient.SetEACL(setEACLPrm) + _, err := internalclient.SetEACL(cmd.Context(), setEACLPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) if containerAwait { @@ -72,7 +72,7 @@ Container ID in EACL table will be substituted with ID from the CLI.`, for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) - res, err := internalclient.EACL(getEACLPrm) + res, err := internalclient.EACL(cmd.Context(), getEACLPrm) if err == nil { // compare binary values because EACL could have been set already table := res.EACL() diff --git a/cmd/frostfs-cli/modules/control/evacuate_shard.go b/cmd/frostfs-cli/modules/control/evacuate_shard.go index b72ff6301..458e4cc0b 100644 --- a/cmd/frostfs-cli/modules/control/evacuate_shard.go +++ b/cmd/frostfs-cli/modules/control/evacuate_shard.go @@ -11,10 +11,11 @@ import ( const ignoreErrorsFlag = "no-errors" var evacuateShardCmd = &cobra.Command{ - Use: "evacuate", - Short: "Evacuate objects from shard", - Long: "Evacuate objects from shard to other shards", - Run: evacuateShard, + Use: "evacuate", + Short: "Evacuate objects from shard", + Long: "Evacuate objects from shard to other shards", + Run: evacuateShard, + Deprecated: "use frostfs-cli control shards evacuation start", } func evacuateShard(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go new file mode 100644 index 000000000..4eb6505cf --- /dev/null +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -0,0 +1,315 @@ +package control + +import ( + "crypto/ecdsa" + "fmt" + "strings" + "sync/atomic" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + "github.com/spf13/cobra" +) + +const ( + awaitFlag = "await" + noProgressFlag = "no-progress" +) + +var evacuationShardCmd = &cobra.Command{ + Use: "evacuation", + Short: "Objects evacuation from shard", + Long: "Objects evacuation from shard to other shards", +} + +var startEvacuationShardCmd = &cobra.Command{ + Use: "start", + Short: "Start evacuate objects from shard", + Long: "Start evacuate objects from shard to other shards", + Run: startEvacuateShard, +} + +var getEvacuationShardStatusCmd = &cobra.Command{ + Use: "status", + Short: "Get evacuate objects from shard status", + Long: "Get evacuate objects from shard to other shards status", + Run: getEvacuateShardStatus, +} + +var stopEvacuationShardCmd = &cobra.Command{ + Use: "stop", + Short: "Stop running evacuate process", + Long: "Stop running evacuate process from shard to other shards", + Run: stopEvacuateShardStatus, +} + +func startEvacuateShard(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) + + req := &control.StartShardEvacuationRequest{ + Body: &control.StartShardEvacuationRequest_Body{ + Shard_ID: getShardIDList(cmd), + IgnoreErrors: ignoreErrors, + }, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.StartShardEvacuationResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.StartShardEvacuation(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "Start evacuate shards failed, rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Shard evacuation has been successfully started.") + + if awaitCompletion, _ := cmd.Flags().GetBool(awaitFlag); awaitCompletion { + noProgress, _ := cmd.Flags().GetBool(noProgressFlag) + waitEvacuateCompletion(cmd, pk, cli, !noProgress, true) + } +} + +func getEvacuateShardStatus(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + req := &control.GetShardEvacuationStatusRequest{ + Body: &control.GetShardEvacuationStatusRequest_Body{}, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.GetShardEvacuationStatusResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.GetShardEvacuationStatus(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "Get evacuate shards status failed, rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + printStatus(cmd, resp) +} + +func stopEvacuateShardStatus(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + req := &control.StopShardEvacuationRequest{ + Body: &control.StopShardEvacuationRequest_Body{}, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.StopShardEvacuationResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.StopShardEvacuation(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "Stop evacuate shards failed, rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + waitEvacuateCompletion(cmd, pk, cli, false, false) + + cmd.Println("Evacuation stopped.") +} + +func waitEvacuateCompletion(cmd *cobra.Command, pk *ecdsa.PrivateKey, cli *clientSDK.Client, printProgress, printCompleted bool) { + const statusPollingInterval = 1 * time.Second + const reportIntervalSeconds = 5 + var resp *control.GetShardEvacuationStatusResponse + reportResponse := new(atomic.Pointer[control.GetShardEvacuationStatusResponse]) + pollingCompleted := make(chan struct{}) + progressReportCompleted := make(chan struct{}) + + go func() { + defer close(progressReportCompleted) + if !printProgress { + return + } + cmd.Printf("Progress will be reported every %d seconds.\n", reportIntervalSeconds) + for { + select { + case <-pollingCompleted: + return + case <-time.After(reportIntervalSeconds * time.Second): + r := reportResponse.Load() + if r == nil || r.GetBody().GetStatus() == control.GetShardEvacuationStatusResponse_Body_COMPLETED { + continue + } + printStatus(cmd, r) + } + } + }() + + for { + req := &control.GetShardEvacuationStatusRequest{ + Body: &control.GetShardEvacuationStatusRequest_Body{}, + } + signRequest(cmd, pk, req) + + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.GetShardEvacuationStatus(client, req) + return err + }) + + reportResponse.Store(resp) + + if err != nil { + commonCmd.ExitOnErr(cmd, "Failed to get evacuate status, rpc error: %w", err) + return + } + if resp.GetBody().GetStatus() != control.GetShardEvacuationStatusResponse_Body_RUNNING { + break + } + + time.Sleep(statusPollingInterval) + } + close(pollingCompleted) + <-progressReportCompleted + if printCompleted { + printCompletedStatusMessage(cmd, resp) + } +} + +func printCompletedStatusMessage(cmd *cobra.Command, resp *control.GetShardEvacuationStatusResponse) { + cmd.Println("Shard evacuation has been completed.") + sb := &strings.Builder{} + appendShardIDs(sb, resp) + appendCounts(sb, resp) + appendError(sb, resp) + appendStartedAt(sb, resp) + appendDuration(sb, resp) + cmd.Println(sb.String()) +} + +func printStatus(cmd *cobra.Command, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetStatus() == control.GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED { + cmd.Println("There is no running or completed evacuation.") + return + } + sb := &strings.Builder{} + appendShardIDs(sb, resp) + appendStatus(sb, resp) + appendCounts(sb, resp) + appendError(sb, resp) + appendStartedAt(sb, resp) + appendDuration(sb, resp) + appendEstimation(sb, resp) + cmd.Println(sb.String()) +} + +func appendEstimation(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetStatus() != control.GetShardEvacuationStatusResponse_Body_RUNNING || + resp.GetBody().GetDuration() == nil || + resp.GetBody().GetTotal() == 0 || + resp.GetBody().GetEvacuated()+resp.GetBody().GetFailed() == 0 { + return + } + + durationSeconds := float64(resp.GetBody().GetDuration().GetSeconds()) + evacuated := float64(resp.GetBody().GetEvacuated() + resp.GetBody().GetFailed()) + avgObjEvacuationTimeSeconds := durationSeconds / evacuated + objectsLeft := float64(resp.GetBody().GetTotal()) - evacuated + leftSeconds := avgObjEvacuationTimeSeconds * objectsLeft + leftMinutes := int(leftSeconds / 60) + + sb.WriteString(fmt.Sprintf(" Estimated time left: %d minutes.", leftMinutes)) +} + +func appendDuration(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetDuration() != nil { + duration := time.Second * time.Duration(resp.GetBody().GetDuration().GetSeconds()) + hour := int(duration.Seconds() / 3600) + minute := int(duration.Seconds()/60) % 60 + second := int(duration.Seconds()) % 60 + sb.WriteString(fmt.Sprintf(" Duration: %02d:%02d:%02d.", hour, minute, second)) + } +} + +func appendStartedAt(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetStartedAt() != nil { + startedAt := time.Unix(resp.GetBody().GetStartedAt().GetValue(), 0).UTC() + sb.WriteString(fmt.Sprintf(" Started at: %s UTC.", startedAt.Format(time.RFC3339))) + } +} + +func appendError(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if len(resp.Body.GetErrorMessage()) > 0 { + sb.WriteString(fmt.Sprintf(" Error: %s.", resp.Body.GetErrorMessage())) + } +} + +func appendStatus(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + var status string + switch resp.GetBody().GetStatus() { + case control.GetShardEvacuationStatusResponse_Body_COMPLETED: + status = "completed" + case control.GetShardEvacuationStatusResponse_Body_RUNNING: + status = "running" + default: + status = "undefined" + } + sb.WriteString(fmt.Sprintf(" Status: %s.", status)) +} + +func appendShardIDs(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + sb.WriteString("Shard IDs: ") + for idx, shardID := range resp.GetBody().GetShard_ID() { + shardIDStr := shard.NewIDFromBytes(shardID).String() + if idx > 0 { + sb.WriteString(", ") + } + sb.WriteString(shardIDStr) + if idx == len(resp.GetBody().GetShard_ID())-1 { + sb.WriteString(".") + } + } +} + +func appendCounts(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + sb.WriteString(fmt.Sprintf(" Evacuated %d object out of %d, failed to evacuate %d objects.", + resp.GetBody().GetEvacuated(), + resp.Body.GetTotal(), + resp.Body.GetFailed())) +} + +func initControlEvacuationShardCmd() { + evacuationShardCmd.AddCommand(startEvacuationShardCmd) + evacuationShardCmd.AddCommand(getEvacuationShardStatusCmd) + evacuationShardCmd.AddCommand(stopEvacuationShardCmd) + + initControlStartEvacuationShardCmd() + initControlFlags(getEvacuationShardStatusCmd) + initControlFlags(stopEvacuationShardCmd) +} + +func initControlStartEvacuationShardCmd() { + initControlFlags(startEvacuationShardCmd) + + flags := startEvacuationShardCmd.Flags() + flags.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") + flags.Bool(shardAllFlag, false, "Process all shards") + flags.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects") + flags.Bool(awaitFlag, false, "Block execution until evacuation is completed") + flags.Bool(noProgressFlag, false, fmt.Sprintf("Print progress if %s provided", awaitFlag)) + + startEvacuationShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) +} diff --git a/cmd/frostfs-cli/modules/control/healthcheck.go b/cmd/frostfs-cli/modules/control/healthcheck.go index 8d18a5c11..097fba540 100644 --- a/cmd/frostfs-cli/modules/control/healthcheck.go +++ b/cmd/frostfs-cli/modules/control/healthcheck.go @@ -1,15 +1,10 @@ package control import ( - "crypto/ecdsa" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" - ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" ) @@ -19,8 +14,8 @@ const ( var healthCheckCmd = &cobra.Command{ Use: "healthcheck", - Short: "Health check of the FrostFS node", - Long: "Health check of the FrostFS node. Checks storage node by default, use --ir flag to work with Inner Ring.", + Short: "Health check for FrostFS storage nodes", + Long: "Health check for FrostFS storage nodes.", Run: healthCheck, } @@ -29,18 +24,18 @@ func initControlHealthCheckCmd() { flags := healthCheckCmd.Flags() flags.Bool(healthcheckIRFlag, false, "Communicate with IR node") + _ = flags.MarkDeprecated(healthcheckIRFlag, "for health check of inner ring nodes, use the 'control ir healthcheck' command instead.") } -func healthCheck(cmd *cobra.Command, _ []string) { - pk := key.Get(cmd) - - cli := getClient(cmd, pk) - +func healthCheck(cmd *cobra.Command, args []string) { if isIR, _ := cmd.Flags().GetBool(healthcheckIRFlag); isIR { - healthCheckIR(cmd, pk, cli) + irHealthCheck(cmd, args) return } + pk := key.Get(cmd) + cli := getClient(cmd, pk) + req := new(control.HealthCheckRequest) req.SetBody(new(control.HealthCheckRequest_Body)) @@ -59,23 +54,3 @@ func healthCheck(cmd *cobra.Command, _ []string) { cmd.Printf("Network status: %s\n", resp.GetBody().GetNetmapStatus()) cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) } - -func healthCheckIR(cmd *cobra.Command, key *ecdsa.PrivateKey, c *client.Client) { - req := new(ircontrol.HealthCheckRequest) - - req.SetBody(new(ircontrol.HealthCheckRequest_Body)) - - err := ircontrolsrv.SignMessage(key, req) - commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) - - var resp *ircontrol.HealthCheckResponse - err = c.ExecRaw(func(client *rawclient.Client) error { - resp, err = ircontrol.HealthCheck(client, req) - return err - }) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - - cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) -} diff --git a/cmd/frostfs-cli/modules/control/ir.go b/cmd/frostfs-cli/modules/control/ir.go index e89dda076..396d5d0a5 100644 --- a/cmd/frostfs-cli/modules/control/ir.go +++ b/cmd/frostfs-cli/modules/control/ir.go @@ -10,6 +10,10 @@ var irCmd = &cobra.Command{ func initControlIRCmd() { irCmd.AddCommand(tickEpochCmd) + irCmd.AddCommand(removeNodeCmd) + irCmd.AddCommand(irHealthCheckCmd) initControlIRTickEpochCmd() + initControlIRRemoveNodeCmd() + initControlIRHealthCheckCmd() } diff --git a/cmd/frostfs-cli/modules/control/ir_healthcheck.go b/cmd/frostfs-cli/modules/control/ir_healthcheck.go new file mode 100644 index 000000000..e70538ce2 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir_healthcheck.go @@ -0,0 +1,44 @@ +package control + +import ( + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + "github.com/spf13/cobra" +) + +var irHealthCheckCmd = &cobra.Command{ + Use: "healthcheck", + Short: "Health check for FrostFS inner ring nodes", + Long: "Health check for FrostFS inner ring nodes.", + Run: irHealthCheck, +} + +func initControlIRHealthCheckCmd() { + initControlFlags(irHealthCheckCmd) +} + +func irHealthCheck(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + cli := getClient(cmd, pk) + + req := new(ircontrol.HealthCheckRequest) + + req.SetBody(new(ircontrol.HealthCheckRequest_Body)) + + err := ircontrolsrv.SignMessage(pk, req) + commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) + + var resp *ircontrol.HealthCheckResponse + err = cli.ExecRaw(func(client *rawclient.Client) error { + resp, err = ircontrol.HealthCheck(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) +} diff --git a/cmd/frostfs-cli/modules/control/ir_remove_node.go b/cmd/frostfs-cli/modules/control/ir_remove_node.go new file mode 100644 index 000000000..f5b968b7f --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir_remove_node.go @@ -0,0 +1,58 @@ +package control + +import ( + "encoding/hex" + "errors" + + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + "github.com/spf13/cobra" +) + +var removeNodeCmd = &cobra.Command{ + Use: "remove-node", + Short: "Forces a node removal from netmap", + Long: "Forces a node removal from netmap via a notary request. It should be executed on other IR nodes as well.", + Run: removeNode, +} + +func initControlIRRemoveNodeCmd() { + initControlFlags(removeNodeCmd) + + flags := removeNodeCmd.Flags() + flags.String("node", "", "Node public key as a hex string") + _ = removeNodeCmd.MarkFlagRequired("node") +} + +func removeNode(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + c := getClient(cmd, pk) + + nodeKeyStr, _ := cmd.Flags().GetString("node") + if len(nodeKeyStr) == 0 { + commonCmd.ExitOnErr(cmd, "parsing node public key: ", errors.New("key cannot be empty")) + } + nodeKey, err := hex.DecodeString(nodeKeyStr) + commonCmd.ExitOnErr(cmd, "can't decode node public key: %w", err) + + req := new(ircontrol.RemoveNodeRequest) + req.SetBody(&ircontrol.RemoveNodeRequest_Body{ + Key: nodeKey, + }) + + commonCmd.ExitOnErr(cmd, "could not sign request: %w", ircontrolsrv.SignMessage(pk, req)) + + var resp *ircontrol.RemoveNodeResponse + err = c.ExecRaw(func(client *rawclient.Client) error { + resp, err = ircontrol.RemoveNode(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Node removed") +} diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 8e7ecff8c..742109673 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -13,13 +13,14 @@ var shardsCmd = &cobra.Command{ func initControlShardsCmd() { shardsCmd.AddCommand(listShardsCmd) shardsCmd.AddCommand(setShardModeCmd) - shardsCmd.AddCommand(evacuateShardCmd) + shardsCmd.AddCommand(evacuationShardCmd) shardsCmd.AddCommand(flushCacheCmd) shardsCmd.AddCommand(doctorCmd) initControlShardsListCmd() initControlSetShardModeCmd() initControlEvacuateShardCmd() + initControlEvacuationShardCmd() initControlFlushCacheCmd() initControlDoctorCmd() } diff --git a/cmd/frostfs-cli/modules/control/util.go b/cmd/frostfs-cli/modules/control/util.go index fdf17244b..5ad675c0e 100644 --- a/cmd/frostfs-cli/modules/control/util.go +++ b/cmd/frostfs-cli/modules/control/util.go @@ -40,7 +40,7 @@ func verifyResponse(cmd *cobra.Command, commonCmd.ExitOnErr(cmd, "", errors.New("missing response signature")) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetScheme(refs.ECDSA_SHA512) sigV2.SetKey(sigControl.GetKey()) diff --git a/cmd/frostfs-cli/modules/netmap/get_epoch.go b/cmd/frostfs-cli/modules/netmap/get_epoch.go index 6e05721ff..a9c2e1f19 100644 --- a/cmd/frostfs-cli/modules/netmap/get_epoch.go +++ b/cmd/frostfs-cli/modules/netmap/get_epoch.go @@ -19,7 +19,7 @@ var getEpochCmd = &cobra.Command{ var prm internalclient.NetworkInfoPrm prm.SetClient(cli) - res, err := internalclient.NetworkInfo(prm) + res, err := internalclient.NetworkInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) netInfo := res.NetworkInfo() diff --git a/cmd/frostfs-cli/modules/netmap/netinfo.go b/cmd/frostfs-cli/modules/netmap/netinfo.go index 17acfd59c..c6ab0b6f6 100644 --- a/cmd/frostfs-cli/modules/netmap/netinfo.go +++ b/cmd/frostfs-cli/modules/netmap/netinfo.go @@ -23,7 +23,7 @@ var netInfoCmd = &cobra.Command{ var prm internalclient.NetworkInfoPrm prm.SetClient(cli) - res, err := internalclient.NetworkInfo(prm) + res, err := internalclient.NetworkInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) netInfo := res.NetworkInfo() diff --git a/cmd/frostfs-cli/modules/netmap/nodeinfo.go b/cmd/frostfs-cli/modules/netmap/nodeinfo.go index 4a94d9e70..3b2113efb 100644 --- a/cmd/frostfs-cli/modules/netmap/nodeinfo.go +++ b/cmd/frostfs-cli/modules/netmap/nodeinfo.go @@ -25,7 +25,7 @@ var nodeInfoCmd = &cobra.Command{ var prm internalclient.NodeInfoPrm prm.SetClient(cli) - res, err := internalclient.NodeInfo(prm) + res, err := internalclient.NodeInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) prettyPrintNodeInfo(cmd, res.NodeInfo()) diff --git a/cmd/frostfs-cli/modules/netmap/root.go b/cmd/frostfs-cli/modules/netmap/root.go index aaa83f12f..006ac6d9f 100644 --- a/cmd/frostfs-cli/modules/netmap/root.go +++ b/cmd/frostfs-cli/modules/netmap/root.go @@ -1,6 +1,7 @@ package netmap import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -14,7 +15,9 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/netmap/snapshot.go b/cmd/frostfs-cli/modules/netmap/snapshot.go index 0878f5ceb..eaaf598b9 100644 --- a/cmd/frostfs-cli/modules/netmap/snapshot.go +++ b/cmd/frostfs-cli/modules/netmap/snapshot.go @@ -19,7 +19,7 @@ var snapshotCmd = &cobra.Command{ var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - res, err := internalclient.NetMapSnapshot(prm) + res, err := internalclient.NetMapSnapshot(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) commonCmd.PrettyPrintNetMap(cmd, res.NetMap(), false) diff --git a/cmd/frostfs-cli/modules/object/delete.go b/cmd/frostfs-cli/modules/object/delete.go index 25d5703de..e4e9cddb8 100644 --- a/cmd/frostfs-cli/modules/object/delete.go +++ b/cmd/frostfs-cli/modules/object/delete.go @@ -65,7 +65,7 @@ func deleteObject(cmd *cobra.Command, _ []string) { Prepare(cmd, &prm) prm.SetAddress(objAddr) - res, err := internalclient.DeleteObject(prm) + res, err := internalclient.DeleteObject(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) tomb := res.Tombstone() diff --git a/cmd/frostfs-cli/modules/object/get.go b/cmd/frostfs-cli/modules/object/get.go index 68e47da6f..3136f086f 100644 --- a/cmd/frostfs-cli/modules/object/get.go +++ b/cmd/frostfs-cli/modules/object/get.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/cheggaaa/pb" "github.com/spf13/cobra" @@ -84,13 +84,13 @@ func getObject(cmd *cobra.Command, _ []string) { p = pb.New64(0) p.Output = cmd.OutOrStdout() prm.SetPayloadWriter(p.NewProxyWriter(payloadWriter)) - prm.SetHeaderCallback(func(o *object.Object) { + prm.SetHeaderCallback(func(o *objectSDK.Object) { p.SetTotal64(int64(o.PayloadSize())) p.Start() }) } - res, err := internalclient.GetObject(prm) + res, err := internalclient.GetObject(cmd.Context(), prm) if p != nil { p.Finish() } diff --git a/cmd/frostfs-cli/modules/object/hash.go b/cmd/frostfs-cli/modules/object/hash.go index c7d734e67..26243e7e7 100644 --- a/cmd/frostfs-cli/modules/object/hash.go +++ b/cmd/frostfs-cli/modules/object/hash.go @@ -75,7 +75,7 @@ func getObjectHash(cmd *cobra.Command, _ []string) { headPrm.SetAddress(objAddr) // get hash of full payload through HEAD (may be user can do it through dedicated command?) - res, err := internalclient.HeadObject(headPrm) + res, err := internalclient.HeadObject(cmd.Context(), headPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) var cs checksum.Checksum @@ -108,7 +108,7 @@ func getObjectHash(cmd *cobra.Command, _ []string) { hashPrm.TZ() } - res, err := internalclient.HashPayloadRanges(hashPrm) + res, err := internalclient.HashPayloadRanges(cmd.Context(), hashPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) hs := res.HashList() diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index 139563e24..db466e588 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -13,7 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -64,7 +64,7 @@ func getObjectHeader(cmd *cobra.Command, _ []string) { prm.SetAddress(objAddr) prm.SetMainOnlyFlag(mainOnly) - res, err := internalclient.HeadObject(prm) + res, err := internalclient.HeadObject(cmd.Context(), prm) if err != nil { if ok := printSplitInfoErr(cmd, err); ok { return @@ -77,7 +77,7 @@ func getObjectHeader(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "", err) } -func saveAndPrintHeader(cmd *cobra.Command, obj *object.Object, filename string) error { +func saveAndPrintHeader(cmd *cobra.Command, obj *objectSDK.Object, filename string) error { bs, err := marshalHeader(cmd, obj) if err != nil { return fmt.Errorf("could not marshal header: %w", err) @@ -97,7 +97,7 @@ func saveAndPrintHeader(cmd *cobra.Command, obj *object.Object, filename string) return printHeader(cmd, obj) } -func marshalHeader(cmd *cobra.Command, hdr *object.Object) ([]byte, error) { +func marshalHeader(cmd *cobra.Command, hdr *objectSDK.Object) ([]byte, error) { toJSON, _ := cmd.Flags().GetBool(commonflags.JSON) toProto, _ := cmd.Flags().GetBool("proto") switch { @@ -138,7 +138,7 @@ func printContainerID(cmd *cobra.Command, recv func() (cid.ID, bool)) { cmd.Printf("CID: %s\n", strID) } -func printHeader(cmd *cobra.Command, obj *object.Object) error { +func printHeader(cmd *cobra.Command, obj *objectSDK.Object) error { printObjectID(cmd, obj.ID) printContainerID(cmd, obj.ContainerID) cmd.Printf("Owner: %s\n", obj.OwnerID()) @@ -150,7 +150,7 @@ func printHeader(cmd *cobra.Command, obj *object.Object) error { cmd.Println("Attributes:") for _, attr := range obj.Attributes() { - if attr.Key() == object.AttributeTimestamp { + if attr.Key() == objectSDK.AttributeTimestamp { cmd.Printf(" %s=%s (%s)\n", attr.Key(), attr.Value(), @@ -163,7 +163,7 @@ func printHeader(cmd *cobra.Command, obj *object.Object) error { if signature := obj.Signature(); signature != nil { cmd.Print("ID signature:\n") - // TODO(@carpawell): #1387 implement and use another approach to avoid conversion + // TODO(@carpawell): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature signature.WriteToV2(&sigV2) @@ -174,7 +174,7 @@ func printHeader(cmd *cobra.Command, obj *object.Object) error { return printSplitHeader(cmd, obj) } -func printSplitHeader(cmd *cobra.Command, obj *object.Object) error { +func printSplitHeader(cmd *cobra.Command, obj *objectSDK.Object) error { if splitID := obj.SplitID(); splitID != nil { cmd.Printf("Split ID: %s\n", splitID) } diff --git a/cmd/frostfs-cli/modules/object/lock.go b/cmd/frostfs-cli/modules/object/lock.go index e6fbedd2c..fa1898586 100644 --- a/cmd/frostfs-cli/modules/object/lock.go +++ b/cmd/frostfs-cli/modules/object/lock.go @@ -104,7 +104,7 @@ var objectLockCmd = &cobra.Command{ Prepare(cmd, &prm) prm.SetHeader(obj) - res, err := internalclient.PutObject(prm) + res, err := internalclient.PutObject(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "Store lock object in FrostFS: %w", err) cmd.Printf("Lock object ID: %s\n", res.ID()) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go new file mode 100644 index 000000000..d2d20dc08 --- /dev/null +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -0,0 +1,348 @@ +package object + +import ( + "context" + "crypto/ecdsa" + "encoding/hex" + "errors" + "fmt" + "strconv" + "sync" + "text/tabwriter" + + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +const ( + verifyPresenceAllFlag = "verify-presence-all" +) + +type objectNodesInfo struct { + containerID cid.ID + objectID oid.ID + relatedObjectIDs []oid.ID + isLock bool +} + +type boolError struct { + value bool + err error +} + +var objectNodesCmd = &cobra.Command{ + Use: "nodes", + Short: "List of nodes where the object is stored", + Long: `List of nodes where the object should be stored and where it is actually stored. + Lock objects must exist on all nodes of the container. + For complex objects, a node is considered to store an object if the node stores at least one part of the complex object. + By default, the actual storage of the object is checked only on the nodes that should store the object. To check all nodes, use the flag --verify-presence-all.`, + Run: objectNodes, +} + +func initObjectNodesCmd() { + commonflags.Init(objectNodesCmd) + + flags := objectNodesCmd.Flags() + + flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + _ = objectGetCmd.MarkFlagRequired(commonflags.CIDFlag) + + flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) + _ = objectGetCmd.MarkFlagRequired(commonflags.OIDFlag) + + flags.Bool("verify-presence-all", false, "Verify the actual presence of the object on all netmap nodes") +} + +func objectNodes(cmd *cobra.Command, _ []string) { + var cnrID cid.ID + var objID oid.ID + readObjectAddress(cmd, &cnrID, &objID) + + pk := key.GetOrGenerate(cmd) + cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) + + objectInfo := getObjectInfo(cmd, cnrID, objID, cli, pk) + + placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli) + + requiredPlacement := getRequiredPlacement(cmd, objectInfo, placementPolicy, netmap) + + actualPlacement := getActualPlacement(cmd, netmap, requiredPlacement, pk, objectInfo) + + printPlacement(cmd, netmap, requiredPlacement, actualPlacement) +} + +func getObjectInfo(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) *objectNodesInfo { + var addrObj oid.Address + addrObj.SetContainer(cnrID) + addrObj.SetObject(objID) + + var prmHead internalclient.HeadObjectPrm + prmHead.SetClient(cli) + prmHead.SetAddress(addrObj) + prmHead.SetRawFlag(true) + + Prepare(cmd, &prmHead) + readSession(cmd, &prmHead, pk, cnrID, objID) + + res, err := internalclient.HeadObject(cmd.Context(), prmHead) + if err == nil { + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + isLock: res.Header().Type() == objectSDK.TypeLock, + } + } + + var errSplitInfo *objectSDK.SplitInfoError + + if !errors.As(err, &errSplitInfo) { + commonCmd.ExitOnErr(cmd, "failed to get object info: %w", err) + return nil + } + + splitInfo := errSplitInfo.SplitInfo() + + if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID); ok { + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + relatedObjectIDs: members, + } + } + + if members, ok := tryGetSplitMembersBySplitID(cmd, splitInfo, cli, cnrID); ok { + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + relatedObjectIDs: members, + } + } + + members := tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnrID, objID) + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + relatedObjectIDs: members, + } +} + +func getPlacementPolicyAndNetmap(cmd *cobra.Command, cnrID cid.ID, cli *client.Client) (placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) { + eg, egCtx := errgroup.WithContext(cmd.Context()) + eg.Go(func() (e error) { + placementPolicy, e = getPlacementPolicy(egCtx, cnrID, cli) + return + }) + eg.Go(func() (e error) { + netmap, e = getNetMap(egCtx, cli) + return + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", eg.Wait()) + return +} + +func getPlacementPolicy(ctx context.Context, cnrID cid.ID, cli *client.Client) (netmapSDK.PlacementPolicy, error) { + var prm internalclient.GetContainerPrm + prm.SetClient(cli) + prm.SetContainer(cnrID) + + res, err := internalclient.GetContainer(ctx, prm) + if err != nil { + return netmapSDK.PlacementPolicy{}, err + } + + return res.Container().PlacementPolicy(), nil +} + +func getNetMap(ctx context.Context, cli *client.Client) (*netmapSDK.NetMap, error) { + var prm internalclient.NetMapSnapshotPrm + prm.SetClient(cli) + + res, err := internalclient.NetMapSnapshot(ctx, prm) + if err != nil { + return nil, err + } + nm := res.NetMap() + return &nm, nil +} + +func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo { + nodes := make(map[uint64]netmapSDK.NodeInfo) + placementBuilder := placement.NewNetworkMapBuilder(netmap) + placement, err := placementBuilder.BuildPlacement(objInfo.containerID, &objInfo.objectID, placementPolicy) + commonCmd.ExitOnErr(cmd, "failed to get required placement: %w", err) + for repIdx, rep := range placement { + numOfReplicas := placementPolicy.ReplicaNumberByIndex(repIdx) + var nodeIdx uint32 + for _, n := range rep { + if !objInfo.isLock && nodeIdx == numOfReplicas { //lock object should be on all container nodes + break + } + nodes[n.Hash()] = n + nodeIdx++ + } + } + + for _, relatedObjID := range objInfo.relatedObjectIDs { + placement, err = placementBuilder.BuildPlacement(objInfo.containerID, &relatedObjID, placementPolicy) + commonCmd.ExitOnErr(cmd, "failed to get required placement for related object: %w", err) + for _, rep := range placement { + for _, n := range rep { + nodes[n.Hash()] = n + } + } + } + + return nodes +} + +func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, + pk *ecdsa.PrivateKey, objInfo *objectNodesInfo) map[uint64]boolError { + result := make(map[uint64]boolError) + resultMtx := &sync.Mutex{} + + var candidates []netmapSDK.NodeInfo + checkAllNodes, _ := cmd.Flags().GetBool(verifyPresenceAllFlag) + if checkAllNodes { + candidates = netmap.Nodes() + } else { + for _, n := range requiredPlacement { + candidates = append(candidates, n) + } + } + + eg, egCtx := errgroup.WithContext(cmd.Context()) + for _, cand := range candidates { + cand := cand + + eg.Go(func() error { + cli, err := createClient(egCtx, cmd, cand, pk) + if err != nil { + resultMtx.Lock() + defer resultMtx.Unlock() + result[cand.Hash()] = boolError{err: err} + return nil + } + + eg.Go(func() error { + var v boolError + v.value, v.err = isObjectStoredOnNode(egCtx, cmd, objInfo.containerID, objInfo.objectID, cli, pk) + resultMtx.Lock() + defer resultMtx.Unlock() + if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { + return nil + } + result[cand.Hash()] = v + return nil + }) + + for _, rObjID := range objInfo.relatedObjectIDs { + rObjID := rObjID + eg.Go(func() error { + var v boolError + v.value, v.err = isObjectStoredOnNode(egCtx, cmd, objInfo.containerID, rObjID, cli, pk) + resultMtx.Lock() + defer resultMtx.Unlock() + if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { + return nil + } + result[cand.Hash()] = v + return nil + }) + } + return nil + }) + } + + commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", eg.Wait()) + return result +} + +func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.NodeInfo, pk *ecdsa.PrivateKey) (*client.Client, error) { + var cli *client.Client + var addresses []string + candidate.IterateNetworkEndpoints(func(s string) bool { + addresses = append(addresses, s) + return false + }) + addresses = append(addresses, candidate.ExternalAddresses()...) + var lastErr error + for _, address := range addresses { + var networkAddr network.Address + lastErr = networkAddr.FromString(address) + if lastErr != nil { + continue + } + cli, lastErr = internalclient.GetSDKClient(ctx, cmd, pk, networkAddr) + if lastErr == nil { + break + } + } + if lastErr != nil { + return nil, lastErr + } + if cli == nil { + return nil, fmt.Errorf("failed to create client: no available endpoint") + } + return cli, nil +} + +func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) (bool, error) { + var addrObj oid.Address + addrObj.SetContainer(cnrID) + addrObj.SetObject(objID) + + var prmHead internalclient.HeadObjectPrm + prmHead.SetClient(cli) + prmHead.SetAddress(addrObj) + + Prepare(cmd, &prmHead) + prmHead.SetTTL(1) + readSession(cmd, &prmHead, pk, cnrID, objID) + + res, err := internalclient.HeadObject(ctx, prmHead) + if err == nil && res != nil { + return true, nil + } + var notFound *apistatus.ObjectNotFound + var removed *apistatus.ObjectAlreadyRemoved + if errors.As(err, ¬Found) || errors.As(err, &removed) { + return false, nil + } + return false, err +} + +func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, actualPlacement map[uint64]boolError) { + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug) + defer func() { + commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", w.Flush()) + }() + fmt.Fprintln(w, "Node ID\tShould contain object\tActually contains object\t") + for _, n := range netmap.Nodes() { + nodeID := hex.EncodeToString(n.PublicKey()) + _, required := requiredPlacement[n.Hash()] + actual, actualExists := actualPlacement[n.Hash()] + actualStr := "" + if actualExists { + if actual.err != nil { + actualStr = fmt.Sprintf("error: %v", actual.err) + } else { + actualStr = strconv.FormatBool(actual.value) + } + } + fmt.Fprintf(w, "%s\t%s\t%s\t\n", nodeID, strconv.FormatBool(required), actualStr) + } +} diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index fe8e9dda9..97bb12dbc 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -16,15 +16,17 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/cheggaaa/pb" "github.com/spf13/cobra" ) const ( - noProgressFlag = "no-progress" - notificationFlag = "notify" + noProgressFlag = "no-progress" + notificationFlag = "notify" + copiesNumberFlag = "copies-number" + prepareLocallyFlag = "prepare-locally" ) var putExpiredOn uint64 @@ -53,9 +55,12 @@ func initObjectPutCmd() { flags.Bool("disable-timestamp", false, "Do not set well-known timestamp attribute") flags.Uint64VarP(&putExpiredOn, commonflags.ExpireAt, "e", 0, "The last active epoch in the life of the object") flags.Bool(noProgressFlag, false, "Do not show progress bar") + flags.Bool(prepareLocallyFlag, false, "Generate object header on the client side (for big object - split locally too)") flags.String(notificationFlag, "", "Object notification in the form of *epoch*:*topic*; '-' topic means using default") flags.Bool(binaryFlag, false, "Deserialize object structure from given file.") + + flags.String(copiesNumberFlag, "", "Number of copies of the object to store within the RPC call") } func putObject(cmd *cobra.Command, _ []string) { @@ -76,7 +81,7 @@ func putObject(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "", fmt.Errorf("can't open file '%s': %w", filename, err)) } var payloadReader io.Reader = f - obj := object.New() + obj := objectSDK.New() if binary { payloadReader, cnr, ownerID = readFilePayload(filename, cmd) @@ -99,7 +104,12 @@ func putObject(cmd *cobra.Command, _ []string) { } var prm internalclient.PutObjectPrm - ReadOrOpenSession(cmd, &prm, pk, cnr, nil) + if prepareLocally, _ := cmd.Flags().GetBool(prepareLocallyFlag); prepareLocally { + prm.SetClient(internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC)) + prm.PrepareLocally() + } else { + ReadOrOpenSession(cmd, &prm, pk, cnr, nil) + } Prepare(cmd, &prm) prm.SetHeader(obj) @@ -116,7 +126,11 @@ func putObject(cmd *cobra.Command, _ []string) { } } - res, err := internalclient.PutObject(prm) + copyNum, err := cmd.Flags().GetString(copiesNumberFlag) + commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) + prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum)) + + res, err := internalclient.PutObject(cmd.Context(), prm) if p != nil { p.Finish() } @@ -126,10 +140,22 @@ func putObject(cmd *cobra.Command, _ []string) { cmd.Printf(" OID: %s\n CID: %s\n", res.ID(), cnr) } +func parseCopyNumber(cmd *cobra.Command, copyNum string) []uint32 { + var cn []uint32 + if len(copyNum) > 0 { + for _, num := range strings.Split(copyNum, ",") { + val, err := strconv.ParseUint(num, 10, 32) + commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) + cn = append(cn, uint32(val)) + } + } + return cn +} + func readFilePayload(filename string, cmd *cobra.Command) (io.Reader, cid.ID, user.ID) { buf, err := os.ReadFile(filename) commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err) - objTemp := object.New() + objTemp := objectSDK.New() // TODO(@acid-ant): #1932 Use streams to marshal/unmarshal payload commonCmd.ExitOnErr(cmd, "can't unmarshal object from given file: %w", objTemp.Unmarshal(buf)) payloadReader := bytes.NewReader(objTemp.Payload()) @@ -148,19 +174,19 @@ func setFilePayloadReader(cmd *cobra.Command, f *os.File, prm *internalclient.Pu p := pb.New64(fi.Size()) p.Output = cmd.OutOrStdout() prm.SetPayloadReader(p.NewProxyReader(f)) - prm.SetHeaderCallback(func(o *object.Object) { p.Start() }) + prm.SetHeaderCallback(func(o *objectSDK.Object) { p.Start() }) return p } -func setBinaryPayloadReader(cmd *cobra.Command, obj *object.Object, prm *internalclient.PutObjectPrm, payloadReader io.Reader) *pb.ProgressBar { +func setBinaryPayloadReader(cmd *cobra.Command, obj *objectSDK.Object, prm *internalclient.PutObjectPrm, payloadReader io.Reader) *pb.ProgressBar { p := pb.New(len(obj.Payload())) p.Output = cmd.OutOrStdout() prm.SetPayloadReader(p.NewProxyReader(payloadReader)) - prm.SetHeaderCallback(func(o *object.Object) { p.Start() }) + prm.SetHeaderCallback(func(o *objectSDK.Object) { p.Start() }) return p } -func getAllObjectAttributes(cmd *cobra.Command) []object.Attribute { +func getAllObjectAttributes(cmd *cobra.Command) []objectSDK.Attribute { attrs, err := parseObjectAttrs(cmd) commonCmd.ExitOnErr(cmd, "can't parse object attributes: %w", err) @@ -179,7 +205,7 @@ func getAllObjectAttributes(cmd *cobra.Command) []object.Attribute { if !expAttrFound { index := len(attrs) - attrs = append(attrs, object.Attribute{}) + attrs = append(attrs, objectSDK.Attribute{}) attrs[index].SetKey(objectV2.SysAttributeExpEpoch) attrs[index].SetValue(expAttrValue) } @@ -187,7 +213,7 @@ func getAllObjectAttributes(cmd *cobra.Command) []object.Attribute { return attrs } -func parseObjectAttrs(cmd *cobra.Command) ([]object.Attribute, error) { +func parseObjectAttrs(cmd *cobra.Command) ([]objectSDK.Attribute, error) { var rawAttrs []string raw := cmd.Flag("attributes").Value.String() @@ -195,7 +221,7 @@ func parseObjectAttrs(cmd *cobra.Command) ([]object.Attribute, error) { rawAttrs = strings.Split(raw, ",") } - attrs := make([]object.Attribute, len(rawAttrs), len(rawAttrs)+2) // name + timestamp attributes + attrs := make([]objectSDK.Attribute, len(rawAttrs), len(rawAttrs)+2) // name + timestamp attributes for i := range rawAttrs { k, v, found := strings.Cut(rawAttrs[i], "=") if !found { @@ -209,23 +235,23 @@ func parseObjectAttrs(cmd *cobra.Command) ([]object.Attribute, error) { if !disableFilename { filename := filepath.Base(cmd.Flag(fileFlag).Value.String()) index := len(attrs) - attrs = append(attrs, object.Attribute{}) - attrs[index].SetKey(object.AttributeFileName) + attrs = append(attrs, objectSDK.Attribute{}) + attrs[index].SetKey(objectSDK.AttributeFileName) attrs[index].SetValue(filename) } disableTime, _ := cmd.Flags().GetBool("disable-timestamp") if !disableTime { index := len(attrs) - attrs = append(attrs, object.Attribute{}) - attrs[index].SetKey(object.AttributeTimestamp) + attrs = append(attrs, objectSDK.Attribute{}) + attrs[index].SetKey(objectSDK.AttributeTimestamp) attrs[index].SetValue(strconv.FormatInt(time.Now().Unix(), 10)) } return attrs, nil } -func parseObjectNotifications(cmd *cobra.Command) (*object.NotificationInfo, error) { +func parseObjectNotifications(cmd *cobra.Command) (*objectSDK.NotificationInfo, error) { const ( separator = ":" useDefaultTopic = "-" @@ -241,7 +267,7 @@ func parseObjectNotifications(cmd *cobra.Command) (*object.NotificationInfo, err return nil, fmt.Errorf("notification must be in the form of: *epoch*%s*topic*, got %s", separator, raw) } - ni := new(object.NotificationInfo) + ni := new(objectSDK.NotificationInfo) epoch, err := strconv.ParseUint(before, 10, 64) if err != nil { diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index a594204f0..591355b95 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -14,7 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -87,7 +87,7 @@ func getObjectRange(cmd *cobra.Command, _ []string) { prm.SetRange(ranges[0]) prm.SetPayloadWriter(out) - _, err = internalclient.PayloadRange(prm) + _, err = internalclient.PayloadRange(cmd.Context(), prm) if err != nil { if ok := printSplitInfoErr(cmd, err); ok { return @@ -102,7 +102,7 @@ func getObjectRange(cmd *cobra.Command, _ []string) { } func printSplitInfoErr(cmd *cobra.Command, err error) bool { - var errSplitInfo *object.SplitInfoError + var errSplitInfo *objectSDK.SplitInfoError ok := errors.As(err, &errSplitInfo) @@ -114,14 +114,14 @@ func printSplitInfoErr(cmd *cobra.Command, err error) bool { return ok } -func printSplitInfo(cmd *cobra.Command, info *object.SplitInfo) { +func printSplitInfo(cmd *cobra.Command, info *objectSDK.SplitInfo) { bs, err := marshalSplitInfo(cmd, info) commonCmd.ExitOnErr(cmd, "can't marshal split info: %w", err) cmd.Println(string(bs)) } -func marshalSplitInfo(cmd *cobra.Command, info *object.SplitInfo) ([]byte, error) { +func marshalSplitInfo(cmd *cobra.Command, info *objectSDK.SplitInfo) ([]byte, error) { toJSON, _ := cmd.Flags().GetBool(commonflags.JSON) toProto, _ := cmd.Flags().GetBool("proto") switch { @@ -146,13 +146,13 @@ func marshalSplitInfo(cmd *cobra.Command, info *object.SplitInfo) ([]byte, error } } -func getRangeList(cmd *cobra.Command) ([]*object.Range, error) { +func getRangeList(cmd *cobra.Command) ([]*objectSDK.Range, error) { v := cmd.Flag("range").Value.String() if len(v) == 0 { return nil, nil } vs := strings.Split(v, ",") - rs := make([]*object.Range, len(vs)) + rs := make([]*objectSDK.Range, len(vs)) for i := range vs { before, after, found := strings.Cut(vs[i], rangeSep) if !found { @@ -176,7 +176,7 @@ func getRangeList(cmd *cobra.Command) ([]*object.Range, error) { return nil, fmt.Errorf("invalid '%s' range: uint64 overflow", vs[i]) } - rs[i] = object.NewRange() + rs[i] = objectSDK.NewRange() rs[i].SetOffset(offset) rs[i].SetLength(length) } diff --git a/cmd/frostfs-cli/modules/object/root.go b/cmd/frostfs-cli/modules/object/root.go index 886153075..c46fc058a 100644 --- a/cmd/frostfs-cli/modules/object/root.go +++ b/cmd/frostfs-cli/modules/object/root.go @@ -1,6 +1,7 @@ package object import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -15,7 +16,9 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { @@ -27,7 +30,8 @@ func init() { objectHeadCmd, objectHashCmd, objectRangeCmd, - objectLockCmd} + objectLockCmd, + objectNodesCmd} Cmd.AddCommand(objectChildCommands...) @@ -44,4 +48,5 @@ func init() { initObjectHashCmd() initObjectRangeCmd() initCommandObjectLock() + initObjectNodesCmd() } diff --git a/cmd/frostfs-cli/modules/object/search.go b/cmd/frostfs-cli/modules/object/search.go index de4a8a3b4..ca5d78bc9 100644 --- a/cmd/frostfs-cli/modules/object/search.go +++ b/cmd/frostfs-cli/modules/object/search.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -61,7 +61,7 @@ func searchObject(cmd *cobra.Command, _ []string) { prm.SetContainerID(cnr) prm.SetFilters(sf) - res, err := internalclient.SearchObjects(prm) + res, err := internalclient.SearchObjects(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) ids := res.IDList() @@ -72,18 +72,18 @@ func searchObject(cmd *cobra.Command, _ []string) { } } -var searchUnaryOpVocabulary = map[string]object.SearchMatchType{ - "NOPRESENT": object.MatchNotPresent, +var searchUnaryOpVocabulary = map[string]objectSDK.SearchMatchType{ + "NOPRESENT": objectSDK.MatchNotPresent, } -var searchBinaryOpVocabulary = map[string]object.SearchMatchType{ - "EQ": object.MatchStringEqual, - "NE": object.MatchStringNotEqual, - "COMMON_PREFIX": object.MatchCommonPrefix, +var searchBinaryOpVocabulary = map[string]objectSDK.SearchMatchType{ + "EQ": objectSDK.MatchStringEqual, + "NE": objectSDK.MatchStringNotEqual, + "COMMON_PREFIX": objectSDK.MatchCommonPrefix, } -func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) { - var fs object.SearchFilters +func parseSearchFilters(cmd *cobra.Command) (objectSDK.SearchFilters, error) { + var fs objectSDK.SearchFilters for i := range searchFilters { words := strings.Fields(searchFilters[i]) @@ -97,7 +97,7 @@ func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) { return nil, fmt.Errorf("could not read attributes filter from file: %w", err) } - subFs := object.NewSearchFilters() + subFs := objectSDK.NewSearchFilters() if err := subFs.UnmarshalJSON(data); err != nil { return nil, fmt.Errorf("could not unmarshal attributes filter from file: %w", err) @@ -138,7 +138,7 @@ func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) { return nil, fmt.Errorf("could not parse object ID: %w", err) } - fs.AddObjectIDFilter(object.MatchStringEqual, id) + fs.AddObjectIDFilter(objectSDK.MatchStringEqual, id) } return fs, nil diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index 204409df3..37e9f74e0 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -16,7 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "github.com/spf13/cobra" @@ -87,7 +87,7 @@ func readObjectAddress(cmd *cobra.Command, cnr *cid.ID, obj *oid.ID) oid.Address func readObjectAddressBin(cmd *cobra.Command, cnr *cid.ID, obj *oid.ID, filename string) oid.Address { buf, err := os.ReadFile(filename) commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err) - objTemp := object.New() + objTemp := objectSDK.New() commonCmd.ExitOnErr(cmd, "can't unmarshal object from given file: %w", objTemp.Unmarshal(buf)) var addr oid.Address @@ -278,7 +278,7 @@ func OpenSessionViaClient(cmd *cobra.Command, dst SessionPrm, cli *client.Client common.PrintVerbose(cmd, "Opening remote session with the node...") - err := sessionCli.CreateSession(&tok, cli, sessionLifetime) + err := sessionCli.CreateSession(cmd.Context(), &tok, cli, sessionLifetime) commonCmd.ExitOnErr(cmd, "open remote session: %w", err) common.PrintVerbose(cmd, "Session successfully opened.") @@ -354,9 +354,9 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, Prepare(cmd, &prmHead) - _, err := internal.HeadObject(prmHead) + _, err := internal.HeadObject(cmd.Context(), prmHead) - var errSplit *object.SplitInfoError + var errSplit *objectSDK.SplitInfoError switch { default: @@ -381,7 +381,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, return tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnr, obj) } -func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *object.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID) ([]oid.ID, bool) { +func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID) ([]oid.ID, bool) { // collect split chain by the descending ease of operations (ease is evaluated heuristically). // If any approach fails, we don't try the next since we assume that it will fail too. @@ -396,7 +396,7 @@ func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *object.Spl prmHead.SetRawFlag(false) // client is already set - res, err := internal.HeadObject(prmHead) + res, err := internal.HeadObject(cmd.Context(), prmHead) if err == nil { children := res.Header().Children() @@ -413,19 +413,19 @@ func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *object.Spl return nil, false } -func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *object.SplitInfo, cli *client.Client, cnr cid.ID) ([]oid.ID, bool) { +func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, cli *client.Client, cnr cid.ID) ([]oid.ID, bool) { if idSplit := splitInfo.SplitID(); idSplit != nil { common.PrintVerbose(cmd, "Collecting split members by split ID...") - var query object.SearchFilters - query.AddSplitIDFilter(object.MatchStringEqual, idSplit) + var query objectSDK.SearchFilters + query.AddSplitIDFilter(objectSDK.MatchStringEqual, idSplit) var prm internal.SearchObjectsPrm prm.SetContainerID(cnr) prm.SetClient(cli) prm.SetFilters(query) - res, err := internal.SearchObjects(prm) + res, err := internal.SearchObjects(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "failed to search objects by split ID: %w", err) parts := res.IDList() @@ -437,7 +437,7 @@ func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *object.SplitInfo return nil, false } -func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, prmHead internal.HeadObjectPrm, cli *client.Client, cnr cid.ID, obj oid.ID) []oid.ID { +func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cli *client.Client, cnr cid.ID, obj oid.ID) []oid.ID { var addrObj oid.Address addrObj.SetContainer(cnr) @@ -463,7 +463,7 @@ func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, p addrObj.SetObject(idMember) prmHead.SetAddress(addrObj) - res, err = internal.HeadObject(prmHead) + res, err = internal.HeadObject(cmd.Context(), prmHead) commonCmd.ExitOnErr(cmd, "failed to read split chain member's header: %w", err) idMember, ok = res.Header().PreviousID() @@ -482,15 +482,15 @@ func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, p common.PrintVerbose(cmd, "Looking for a linking object...") - var query object.SearchFilters - query.AddParentIDFilter(object.MatchStringEqual, obj) + var query objectSDK.SearchFilters + query.AddParentIDFilter(objectSDK.MatchStringEqual, obj) var prmSearch internal.SearchObjectsPrm prmSearch.SetClient(cli) prmSearch.SetContainerID(cnr) prmSearch.SetFilters(query) - resSearch, err := internal.SearchObjects(prmSearch) + resSearch, err := internal.SearchObjects(cmd.Context(), prmSearch) commonCmd.ExitOnErr(cmd, "failed to find object children: %w", err) list := resSearch.IDList() diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index 341681f5b..53f6e8bc4 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -1,10 +1,12 @@ package session import ( + "context" "fmt" "os" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -28,10 +30,12 @@ var createCmd = &cobra.Command{ Use: "create", Short: "Create session token", Run: createSession, - PersistentPreRun: func(cmd *cobra.Command, _ []string) { + PersistentPreRun: func(cmd *cobra.Command, args []string) { _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, cmd.Flags().Lookup(commonflags.Account)) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { @@ -64,7 +68,7 @@ func createSession(cmd *cobra.Command, _ []string) { var tok session.Object - err = CreateSession(&tok, c, lifetime) + err = CreateSession(cmd.Context(), &tok, c, lifetime) commonCmd.ExitOnErr(cmd, "can't create session: %w", err) var data []byte @@ -86,11 +90,11 @@ func createSession(cmd *cobra.Command, _ []string) { // number of epochs. // // Fills ID, lifetime and session key. -func CreateSession(dst *session.Object, c *client.Client, lifetime uint64) error { +func CreateSession(ctx context.Context, dst *session.Object, c *client.Client, lifetime uint64) error { var netInfoPrm internalclient.NetworkInfoPrm netInfoPrm.SetClient(c) - ni, err := internalclient.NetworkInfo(netInfoPrm) + ni, err := internalclient.NetworkInfo(ctx, netInfoPrm) if err != nil { return fmt.Errorf("can't fetch network info: %w", err) } @@ -102,7 +106,7 @@ func CreateSession(dst *session.Object, c *client.Client, lifetime uint64) error sessionPrm.SetClient(c) sessionPrm.SetExp(exp) - sessionRes, err := internalclient.CreateSession(sessionPrm) + sessionRes, err := internalclient.CreateSession(ctx, sessionPrm) if err != nil { return fmt.Errorf("can't open session: %w", err) } diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index 707a4d8ee..4ac1ed198 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -49,24 +50,29 @@ func add(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + req := new(tree.AddRequest) req.Body = &tree.AddRequest_Body{ ContainerId: rawCID, TreeId: tid, ParentId: pid, Meta: meta, - BearerToken: nil, // TODO: #1891 add token handling + BearerToken: bt, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.Add(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to cal add: %w", err) cmd.Println("Node ID: ", resp.Body.NodeId) } diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index e83408a89..ea815dbfe 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -10,7 +10,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -53,7 +53,7 @@ func addByPath(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -62,23 +62,27 @@ func addByPath(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "meta data parsing: %w", err) path, _ := cmd.Flags().GetString(pathFlagKey) - // pAttr, _ := cmd.Flags().GetString(pathAttributeFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } req := new(tree.AddByPathRequest) req.Body = &tree.AddByPathRequest_Body{ ContainerId: rawCID, TreeId: tid, - PathAttribute: object.AttributeFileName, + PathAttribute: objectSDK.AttributeFileName, // PathAttribute: pAttr, Path: strings.Split(path, "/"), Meta: meta, - BearerToken: nil, // TODO: #1891 add token handling + BearerToken: bt, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.AddByPath(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to addByPath %w", err) cmd.Printf("Parent ID: %d\n", resp.GetBody().GetParentId()) diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index f25bff166..4f4f54657 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -5,10 +5,11 @@ import ( "strings" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "github.com/spf13/viper" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -26,10 +27,12 @@ func _client(ctx context.Context) (tree.TreeServiceClient, error) { opts := []grpc.DialOption{ grpc.WithBlock(), grpc.WithChainUnaryInterceptor( - tracing.NewGRPCUnaryClientInteceptor(), + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( - tracing.NewGRPCStreamClientInterceptor(), + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), ), } diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index 75acbaedf..f239066cd 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -10,7 +10,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -53,31 +53,35 @@ func getByPath(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) latestOnly, _ := cmd.Flags().GetBool(latestOnlyFlagKey) path, _ := cmd.Flags().GetString(pathFlagKey) - // pAttr, _ := cmd.Flags().GetString(pathAttributeFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } req := new(tree.GetNodeByPathRequest) req.Body = &tree.GetNodeByPathRequest_Body{ ContainerId: rawCID, TreeId: tid, - PathAttribute: object.AttributeFileName, + PathAttribute: objectSDK.AttributeFileName, // PathAttribute: pAttr, Path: strings.Split(path, "/"), LatestOnly: latestOnly, AllAttributes: true, - BearerToken: nil, // TODO: #1891 add token handling + BearerToken: bt, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.GetNodeByPath(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call getNodeByPath: %w", err) nn := resp.GetBody().GetNodes() if len(nn) == 0 { diff --git a/cmd/frostfs-cli/modules/tree/get_op_log.go b/cmd/frostfs-cli/modules/tree/get_op_log.go new file mode 100644 index 000000000..6efa76133 --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/get_op_log.go @@ -0,0 +1,83 @@ +package tree + +import ( + "crypto/sha256" + "errors" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var getOpLogCmd = &cobra.Command{ + Use: "get-op-log", + Short: "Get logged operations starting with some height", + Run: getOpLog, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initGetOpLogCmd() { + commonflags.Init(getOpLogCmd) + initCTID(getOpLogCmd) + + ff := getOpLogCmd.Flags() + ff.Uint64(heightFlagKey, 0, "Height to start with") + ff.Uint64(countFlagKey, 10, "Logged operations count") + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func getOpLog(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + + cidRaw, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidRaw) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + height, _ := cmd.Flags().GetUint64(heightFlagKey) + count, _ := cmd.Flags().GetUint64(countFlagKey) + + req := &tree.GetOpLogRequest{ + Body: &tree.GetOpLogRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + Height: height, + Count: count, + }, + } + + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) + + resp, err := cli.GetOpLog(ctx, req) + commonCmd.ExitOnErr(cmd, "get op log: %w", err) + + opLogResp, err := resp.Recv() + for ; err == nil; opLogResp, err = resp.Recv() { + o := opLogResp.GetBody().GetOperation() + + cmd.Println("Parent ID: ", o.GetParentId()) + + cmd.Println("\tChild ID: ", o.GetChildId()) + cmd.Printf("\tMeta: %s\n", o.GetMeta()) + } + if !errors.Is(err, io.EOF) { + commonCmd.ExitOnErr(cmd, "get op log response stream: %w", err) + } +} diff --git a/cmd/frostfs-cli/modules/tree/healthcheck.go b/cmd/frostfs-cli/modules/tree/healthcheck.go new file mode 100644 index 000000000..f0506467e --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/healthcheck.go @@ -0,0 +1,43 @@ +package tree + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + "github.com/spf13/cobra" +) + +var healthcheckCmd = &cobra.Command{ + Use: "healthcheck", + Short: "Check tree service availability", + Run: healthcheck, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initHealthcheckCmd() { + commonflags.Init(healthcheckCmd) + ff := healthcheckCmd.Flags() + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func healthcheck(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) + + req := &tree.HealthcheckRequest{ + Body: &tree.HealthcheckRequest_Body{}, + } + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) + + _, err = cli.Healthcheck(ctx, req) + commonCmd.ExitOnErr(cmd, "failed to call healthcheck: %w", err) + + common.PrintVerbose(cmd, "Successful healthcheck invocation.") +} diff --git a/cmd/frostfs-cli/modules/tree/list.go b/cmd/frostfs-cli/modules/tree/list.go index 8e4d2bd4c..a25d066d5 100644 --- a/cmd/frostfs-cli/modules/tree/list.go +++ b/cmd/frostfs-cli/modules/tree/list.go @@ -41,7 +41,7 @@ func list(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -52,10 +52,10 @@ func list(cmd *cobra.Command, _ []string) { }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.TreeList(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call treeList %w", err) for _, treeID := range resp.GetBody().GetIds() { cmd.Println(treeID) diff --git a/cmd/frostfs-cli/modules/tree/move.go b/cmd/frostfs-cli/modules/tree/move.go new file mode 100644 index 000000000..84b2fb80e --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/move.go @@ -0,0 +1,107 @@ +package tree + +import ( + "crypto/sha256" + "errors" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var moveCmd = &cobra.Command{ + Use: "move", + Short: "Move node", + Run: move, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initMoveCmd() { + commonflags.Init(moveCmd) + initCTID(moveCmd) + + ff := moveCmd.Flags() + ff.Uint64(nodeIDFlagKey, 0, "Node ID.") + ff.Uint64(parentIDFlagKey, 0, "Parent ID.") + + _ = getSubtreeCmd.MarkFlagRequired(nodeIDFlagKey) + _ = getSubtreeCmd.MarkFlagRequired(parentIDFlagKey) + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func move(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + cidString, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidString) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + pid, _ := cmd.Flags().GetUint64(parentIDFlagKey) + nid, _ := cmd.Flags().GetUint64(nodeIDFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + + subTreeReq := &tree.GetSubTreeRequest{ + Body: &tree.GetSubTreeRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + RootId: nid, + Depth: 1, + BearerToken: bt, + }, + } + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(subTreeReq, pk)) + resp, err := cli.GetSubTree(ctx, subTreeReq) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + + var meta []*tree.KeyValue + subtreeResp, err := resp.Recv() + for ; err == nil; subtreeResp, err = resp.Recv() { + meta = subtreeResp.GetBody().GetMeta() + } + if !errors.Is(err, io.EOF) { + commonCmd.ExitOnErr(cmd, "failed to read getSubTree response stream: %w", err) + } + var metaErr error + if len(meta) == 0 { + metaErr = errors.New("no meta for given node ID") + } + commonCmd.ExitOnErr(cmd, "unexpected rpc call result: %w", metaErr) + + req := &tree.MoveRequest{ + Body: &tree.MoveRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + ParentId: pid, + NodeId: nid, + Meta: meta, + }, + } + + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) + + _, err = cli.Move(ctx, req) + commonCmd.ExitOnErr(cmd, "failed to call move: %w", err) + common.PrintVerbose(cmd, "Successful move invocation.") +} diff --git a/cmd/frostfs-cli/modules/tree/remove.go b/cmd/frostfs-cli/modules/tree/remove.go new file mode 100644 index 000000000..74e9d9749 --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/remove.go @@ -0,0 +1,74 @@ +package tree + +import ( + "crypto/sha256" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var removeCmd = &cobra.Command{ + Use: "remove", + Short: "Remove node", + Run: remove, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initRemoveCmd() { + commonflags.Init(removeCmd) + initCTID(removeCmd) + + ff := removeCmd.Flags() + ff.Uint64(nodeIDFlagKey, 0, "Node ID.") + + _ = getSubtreeCmd.MarkFlagRequired(nodeIDFlagKey) + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func remove(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + cidString, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidString) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + + nid, _ := cmd.Flags().GetUint64(nodeIDFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + req := &tree.RemoveRequest{ + Body: &tree.RemoveRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + NodeId: nid, + BearerToken: bt, + }, + } + + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) + + _, err = cli.Remove(ctx, req) + commonCmd.ExitOnErr(cmd, "failed to call remove: %w", err) + common.PrintVerbose(cmd, "Successful remove invocation.") +} diff --git a/cmd/frostfs-cli/modules/tree/root.go b/cmd/frostfs-cli/modules/tree/root.go index c70e6f5e2..701a78f2a 100644 --- a/cmd/frostfs-cli/modules/tree/root.go +++ b/cmd/frostfs-cli/modules/tree/root.go @@ -15,16 +15,28 @@ func init() { Cmd.AddCommand(getByPathCmd) Cmd.AddCommand(addByPathCmd) Cmd.AddCommand(listCmd) + Cmd.AddCommand(healthcheckCmd) + Cmd.AddCommand(moveCmd) + Cmd.AddCommand(removeCmd) + Cmd.AddCommand(getSubtreeCmd) + Cmd.AddCommand(getOpLogCmd) initAddCmd() initGetByPathCmd() initAddByPathCmd() initListCmd() + initHealthcheckCmd() + initMoveCmd() + initRemoveCmd() + initGetSubtreeCmd() + initGetOpLogCmd() } const ( treeIDFlagKey = "tid" parentIDFlagKey = "pid" + nodeIDFlagKey = "nid" + rootIDFlagKey = "root" metaFlagKey = "meta" @@ -32,6 +44,12 @@ const ( pathAttributeFlagKey = "pattr" latestOnlyFlagKey = "latest" + + bearerFlagKey = "bearer" + + heightFlagKey = "height" + countFlagKey = "count" + depthFlagKey = "depth" ) func initCTID(cmd *cobra.Command) { @@ -42,4 +60,6 @@ func initCTID(cmd *cobra.Command) { ff.String(treeIDFlagKey, "", "Tree ID") _ = cmd.MarkFlagRequired(treeIDFlagKey) + + ff.String(bearerFlagKey, "", "Path to bearer token") } diff --git a/cmd/frostfs-cli/modules/tree/subtree.go b/cmd/frostfs-cli/modules/tree/subtree.go new file mode 100644 index 000000000..64cb351ec --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/subtree.go @@ -0,0 +1,101 @@ +package tree + +import ( + "crypto/sha256" + "errors" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var getSubtreeCmd = &cobra.Command{ + Use: "get-subtree", + Short: "Get subtree", + Run: getSubTree, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initGetSubtreeCmd() { + commonflags.Init(getSubtreeCmd) + initCTID(getSubtreeCmd) + + ff := getSubtreeCmd.Flags() + ff.Uint64(rootIDFlagKey, 0, "Root ID to traverse from.") + ff.Uint32(depthFlagKey, 10, "Traversal depth.") + + _ = getSubtreeCmd.MarkFlagRequired(commonflags.CIDFlag) + _ = getSubtreeCmd.MarkFlagRequired(treeIDFlagKey) + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func getSubTree(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + cidString, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidString) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + + rid, _ := cmd.Flags().GetUint64(rootIDFlagKey) + + depth, _ := cmd.Flags().GetUint32(depthFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + + req := &tree.GetSubTreeRequest{ + Body: &tree.GetSubTreeRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + RootId: rid, + Depth: depth, + BearerToken: bt, + }, + } + + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) + + resp, err := cli.GetSubTree(ctx, req) + commonCmd.ExitOnErr(cmd, "failed to call getSubTree: %w", err) + + subtreeResp, err := resp.Recv() + for ; err == nil; subtreeResp, err = resp.Recv() { + b := subtreeResp.GetBody() + + cmd.Printf("Node ID: %d\n", b.GetNodeId()) + + cmd.Println("\tParent ID: ", b.GetParentId()) + cmd.Println("\tTimestamp: ", b.GetTimestamp()) + + if meta := b.GetMeta(); len(meta) > 0 { + cmd.Println("\tMeta pairs: ") + for _, kv := range meta { + cmd.Printf("\t\t%s: %s\n", kv.GetKey(), string(kv.GetValue())) + } + } + } + if !errors.Is(err, io.EOF) { + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + } +} diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index c4787f965..54c7d18e3 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -60,6 +60,13 @@ func watchForSignal(cancel func()) { if err != nil { log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) } + pprofCmp.reload() + metricsCmp.reload() + log.Info(logs.FrostFSIRReloadExtraWallets) + err = innerRing.SetExtraWallets(cfg) + if err != nil { + log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) + } log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) case syscall.SIGTERM, syscall.SIGINT: log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index 007bb8964..a7fe8d563 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -51,9 +51,8 @@ func setControlDefaults(cfg *viper.Viper) { func setFeeDefaults(cfg *viper.Viper) { // extra fee values for working mode without notary contract - cfg.SetDefault("fee.main_chain", 5000_0000) // 0.5 Fixed8 - cfg.SetDefault("fee.side_chain", 2_0000_0000) // 2.0 Fixed8 - cfg.SetDefault("fee.named_container_register", 25_0000_0000) // 25.0 Fixed8 + cfg.SetDefault("fee.main_chain", 5000_0000) // 0.5 Fixed8 + cfg.SetDefault("fee.side_chain", 2_0000_0000) // 2.0 Fixed8 } func setEmitDefaults(cfg *viper.Viper) { diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go new file mode 100644 index 000000000..3a6d77d84 --- /dev/null +++ b/cmd/frostfs-ir/httpcomponent.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "net/http" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" + "go.uber.org/zap" +) + +type httpComponent struct { + srv *httputil.Server + address string + name string + handler http.Handler + shutdownDur time.Duration + enabled bool +} + +const ( + enabledKeyPostfix = ".enabled" + addressKeyPostfix = ".address" + shutdownTimeoutKeyPostfix = ".shutdown_timeout" +) + +func (c *httpComponent) init() { + log.Info(fmt.Sprintf("init %s", c.name)) + c.enabled = cfg.GetBool(c.name + enabledKeyPostfix) + c.address = cfg.GetString(c.name + addressKeyPostfix) + c.shutdownDur = cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) + + if c.enabled { + c.srv = httputil.New( + httputil.HTTPSrvPrm{ + Address: c.address, + Handler: c.handler, + }, + httputil.WithShutdownTimeout(c.shutdownDur), + ) + } else { + log.Info(fmt.Sprintf("%s is disabled, skip", c.name)) + c.srv = nil + } +} + +func (c *httpComponent) start() { + if c.srv != nil { + log.Info(fmt.Sprintf("start %s", c.name)) + wg.Add(1) + go func() { + defer wg.Done() + exitErr(c.srv.Serve()) + }() + } +} + +func (c *httpComponent) shutdown() error { + if c.srv != nil { + log.Info(fmt.Sprintf("shutdown %s", c.name)) + return c.srv.Shutdown() + } + return nil +} + +func (c *httpComponent) needReload() bool { + enabled := cfg.GetBool(c.name + enabledKeyPostfix) + address := cfg.GetString(c.name + addressKeyPostfix) + dur := cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) + return enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) +} + +func (c *httpComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + if c.needReload() { + log.Info(fmt.Sprintf("%s config updated", c.name)) + if err := c.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), + ) + } else { + c.init() + c.start() + } + } +} diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 932b90404..70199b094 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -4,16 +4,13 @@ import ( "context" "flag" "fmt" - "net/http" "os" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" - httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -29,15 +26,16 @@ const ( ) var ( - wg = new(sync.WaitGroup) - intErr = make(chan error) // internal inner ring errors - logPrm = new(logger.Prm) - innerRing *innerring.Server - httpServers []*httputil.Server - log *logger.Logger - cfg *viper.Viper - configFile *string - configDir *string + wg = new(sync.WaitGroup) + intErr = make(chan error) // internal inner ring errors + logPrm = new(logger.Prm) + innerRing *innerring.Server + pprofCmp *pprofComponent + metricsCmp *httpComponent + log *logger.Logger + cfg *viper.Viper + configFile *string + configDir *string ) func exitErr(err error) { @@ -63,6 +61,7 @@ func main() { cfg, err = newConfig() exitErr(err) + logPrm.MetricsNamespace = "frostfs_ir" err = logPrm.SetLevelString( cfg.GetString("logger.level"), ) @@ -73,19 +72,17 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) - initHTTPServers(cfg) + pprofCmp = newPprofComponent() + pprofCmp.init() + + metricsCmp = newMetricsComponent() + metricsCmp.init() innerRing, err = innerring.New(ctx, log, cfg, intErr) exitErr(err) - // start HTTP servers - for _, srv := range httpServers { - wg.Add(1) - go func(srv *httputil.Server) { - exitErr(srv.Serve()) - wg.Done() - }(srv) - } + pprofCmp.start() + metricsCmp.start() // start inner ring err = innerRing.Start(ctx, intErr) @@ -103,54 +100,16 @@ func main() { log.Info(logs.FrostFSIRApplicationStopped) } -func initHTTPServers(cfg *viper.Viper) { - items := []struct { - cfgPrefix string - handler func() http.Handler - }{ - {"pprof", httputil.Handler}, - {"prometheus", promhttp.Handler}, +func shutdown() { + innerRing.Stop() + if err := metricsCmp.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), + ) } - - httpServers = make([]*httputil.Server, 0, len(items)) - - for _, item := range items { - if !cfg.GetBool(item.cfgPrefix + ".enabled") { - log.Info(item.cfgPrefix + " is disabled, skip") - continue - } - - addr := cfg.GetString(item.cfgPrefix + ".address") - - var prm httputil.HTTPSrvPrm - - prm.Address = addr - prm.Handler = item.handler() - - httpServers = append(httpServers, - httputil.New(prm, - httputil.WithShutdownTimeout( - cfg.GetDuration(item.cfgPrefix+".shutdown_timeout"), - ), - ), + if err := pprofCmp.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), ) } } - -func shutdown() { - innerRing.Stop() - - // shut down HTTP servers - for _, srv := range httpServers { - wg.Add(1) - go func(srv *httputil.Server) { - err := srv.Shutdown() - if err != nil { - log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, - zap.String("error", err.Error()), - ) - } - wg.Done() - }(srv) - } -} diff --git a/cmd/frostfs-ir/metrics.go b/cmd/frostfs-ir/metrics.go new file mode 100644 index 000000000..dd982b780 --- /dev/null +++ b/cmd/frostfs-ir/metrics.go @@ -0,0 +1,12 @@ +package main + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" +) + +func newMetricsComponent() *httpComponent { + return &httpComponent{ + name: "prometheus", + handler: metrics.Handler(), + } +} diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go new file mode 100644 index 000000000..d67c463fc --- /dev/null +++ b/cmd/frostfs-ir/pprof.go @@ -0,0 +1,68 @@ +package main + +import ( + "fmt" + "runtime" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" + "go.uber.org/zap" +) + +type pprofComponent struct { + httpComponent + blockRate int + mutexRate int +} + +const ( + pprofBlockRateKey = "pprof.block_rate" + pprofMutexRateKey = "pprof.mutex_rate" +) + +func newPprofComponent() *pprofComponent { + return &pprofComponent{ + httpComponent: httpComponent{ + name: "pprof", + handler: httputil.Handler(), + }, + } +} + +func (c *pprofComponent) init() { + c.httpComponent.init() + + if c.enabled { + c.blockRate = cfg.GetInt(pprofBlockRateKey) + c.mutexRate = cfg.GetInt(pprofMutexRateKey) + runtime.SetBlockProfileRate(c.blockRate) + runtime.SetMutexProfileFraction(c.mutexRate) + } else { + c.blockRate = 0 + c.mutexRate = 0 + runtime.SetBlockProfileRate(0) + runtime.SetMutexProfileFraction(0) + } +} + +func (c *pprofComponent) needReload() bool { + blockRate := cfg.GetInt(pprofBlockRateKey) + mutexRate := cfg.GetInt(pprofMutexRateKey) + return c.httpComponent.needReload() || + c.enabled && (c.blockRate != blockRate || c.mutexRate != mutexRate) +} + +func (c *pprofComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + if c.needReload() { + log.Info(fmt.Sprintf("%s config updated", c.name)) + if err := c.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error())) + return + } + + c.init() + c.start() + } +} diff --git a/cmd/frostfs-lens/internal/blobovnicza/inspect.go b/cmd/frostfs-lens/internal/blobovnicza/inspect.go index 13442a4b8..b1a6e3fd2 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/inspect.go +++ b/cmd/frostfs-lens/internal/blobovnicza/inspect.go @@ -3,7 +3,7 @@ package blobovnicza import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -38,7 +38,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { data := res.Object() - var o object.Object + var o objectSDK.Object common.ExitOnErr(cmd, common.Errf("could not unmarshal object: %w", o.Unmarshal(data)), ) diff --git a/cmd/frostfs-lens/internal/blobovnicza/list.go b/cmd/frostfs-lens/internal/blobovnicza/list.go index 67242a7d1..d327dbc41 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/list.go +++ b/cmd/frostfs-lens/internal/blobovnicza/list.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "fmt" "io" @@ -33,6 +34,6 @@ func listFunc(cmd *cobra.Command, _ []string) { blz := openBlobovnicza(cmd) defer blz.Close() - err := blobovnicza.IterateAddresses(blz, wAddr) + err := blobovnicza.IterateAddresses(context.Background(), blz, wAddr) common.ExitOnErr(cmd, common.Errf("blobovnicza iterator failure: %w", err)) } diff --git a/cmd/frostfs-lens/internal/meta/inspect.go b/cmd/frostfs-lens/internal/meta/inspect.go index bc7f28a3a..de0f24aeb 100644 --- a/cmd/frostfs-lens/internal/meta/inspect.go +++ b/cmd/frostfs-lens/internal/meta/inspect.go @@ -7,7 +7,7 @@ import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -49,7 +49,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { prm.SetAddress(addr) prm.SetRaw(true) - siErr := new(object.SplitInfoError) + siErr := new(objectSDK.SplitInfoError) res, err := db.Get(cmd.Context(), prm) if errors.As(err, &siErr) { diff --git a/cmd/frostfs-lens/internal/meta/list-garbage.go b/cmd/frostfs-lens/internal/meta/list-garbage.go index 3ab9a8f88..61b10ca1f 100644 --- a/cmd/frostfs-lens/internal/meta/list-garbage.go +++ b/cmd/frostfs-lens/internal/meta/list-garbage.go @@ -28,6 +28,6 @@ func listGarbageFunc(cmd *cobra.Command, _ []string) { return nil }) - err := db.IterateOverGarbage(garbPrm) + err := db.IterateOverGarbage(cmd.Context(), garbPrm) common.ExitOnErr(cmd, common.Errf("could not iterate over garbage bucket: %w", err)) } diff --git a/cmd/frostfs-lens/internal/meta/list-graveyard.go b/cmd/frostfs-lens/internal/meta/list-graveyard.go index db90513eb..19a93691c 100644 --- a/cmd/frostfs-lens/internal/meta/list-graveyard.go +++ b/cmd/frostfs-lens/internal/meta/list-graveyard.go @@ -33,6 +33,6 @@ func listGraveyardFunc(cmd *cobra.Command, _ []string) { return nil }) - err := db.IterateOverGraveyard(gravePrm) + err := db.IterateOverGraveyard(cmd.Context(), gravePrm) common.ExitOnErr(cmd, common.Errf("could not iterate over graveyard bucket: %w", err)) } diff --git a/cmd/frostfs-lens/internal/printers.go b/cmd/frostfs-lens/internal/printers.go index a232409d6..dd73a5552 100644 --- a/cmd/frostfs-lens/internal/printers.go +++ b/cmd/frostfs-lens/internal/printers.go @@ -4,14 +4,14 @@ import ( "os" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) // PrintObjectHeader prints passed object's header fields via // the passed cobra command. Does nothing with the payload. -func PrintObjectHeader(cmd *cobra.Command, h object.Object) { +func PrintObjectHeader(cmd *cobra.Command, h objectSDK.Object) { cmd.Println("Version:", h.Version()) cmd.Println("Type:", h.Type()) printContainerID(cmd, h.ContainerID) diff --git a/cmd/frostfs-lens/internal/writecache/inspect.go b/cmd/frostfs-lens/internal/writecache/inspect.go index 91f12ed0f..7d3c8ab22 100644 --- a/cmd/frostfs-lens/internal/writecache/inspect.go +++ b/cmd/frostfs-lens/internal/writecache/inspect.go @@ -3,7 +3,7 @@ package writecache import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -27,7 +27,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { data, err := writecache.Get(db, []byte(vAddress)) common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err)) - var o object.Object + var o objectSDK.Object common.ExitOnErr(cmd, common.Errf("could not unmarshal object: %w", o.Unmarshal(data))) common.PrintObjectHeader(cmd, o) diff --git a/cmd/frostfs-node/accounting.go b/cmd/frostfs-node/accounting.go index 6a35f37d0..d04f34ff1 100644 --- a/cmd/frostfs-node/accounting.go +++ b/cmd/frostfs-node/accounting.go @@ -21,10 +21,8 @@ func initAccountingService(ctx context.Context, c *cfg) { server := accountingTransportGRPC.New( accountingService.NewSignService( &c.key.PrivateKey, - accountingService.NewResponseService( - accountingService.NewExecutionService( - accounting.NewExecutor(balanceMorphWrapper), - ), + accountingService.NewExecutionService( + accounting.NewExecutor(balanceMorphWrapper), c.respSvc, ), ), diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index dfbaf3525..bae38b299 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -25,19 +25,18 @@ type valueWithTime[V any] struct { } type locker struct { - mtx *sync.Mutex + mtx sync.Mutex waiters int // not protected by mtx, must used outer mutex to update concurrently } type keyLocker[K comparable] struct { lockers map[K]*locker - lockersMtx *sync.Mutex + lockersMtx sync.Mutex } func newKeyLocker[K comparable]() *keyLocker[K] { return &keyLocker[K]{ - lockers: make(map[K]*locker), - lockersMtx: &sync.Mutex{}, + lockers: make(map[K]*locker), } } @@ -53,7 +52,6 @@ func (l *keyLocker[K]) LockKey(key K) { } locker := &locker{ - mtx: &sync.Mutex{}, waiters: 1, } locker.mtx.Lock() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 987d27fcd..283cf501a 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -11,12 +11,11 @@ import ( "path/filepath" "strings" "sync" - atomicstd "sync/atomic" + "sync/atomic" "syscall" "time" netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient" contractsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/contracts" @@ -37,6 +36,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + lsmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" shardmode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -60,6 +60,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -68,7 +69,6 @@ import ( neogoutil "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -98,6 +98,7 @@ type applicationConfiguration struct { errorThreshold uint32 shardPoolSize uint32 shards []shardCfg + lowMem bool } } @@ -200,6 +201,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { a.EngineCfg.errorThreshold = engineconfig.ShardErrorThreshold(c) a.EngineCfg.shardPoolSize = engineconfig.ShardPoolSize(c) + a.EngineCfg.lowMem = engineconfig.EngineLowMemoryConsumption(c) return engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { return a.updateShardConfig(c, sc) }) } @@ -321,20 +323,22 @@ func (a *applicationConfiguration) setGCConfig(newConfig *shardCfg, oldConfig *s // helpers and fields. type internals struct { done chan struct{} + ctxCancel func() internalErr chan error // channel for internal application errors at runtime appCfg *config.Config log *logger.Logger - wg *sync.WaitGroup + wg sync.WaitGroup workers []worker closers []closer apiVersion version.Version healthStatus *atomic.Int32 // is node under maintenance - isMaintenance atomic.Bool + isMaintenance atomic.Bool + isOnlineCandidate bool } // starts node's maintenance. @@ -373,7 +377,7 @@ type shared struct { ownerIDFromKey user.ID // user ID calculated from key // current network map - netMap atomicstd.Value // type netmap.NetMap + netMap atomic.Value // type netmap.NetMap netMapSource netmapCore.Source cnrClient *containerClient.Client @@ -542,6 +546,8 @@ func initCfg(appCfg *config.Config) *cfg { logPrm, err := c.loggerPrm() fatalOnErr(err) + logPrm.MetricsNamespace = "frostfs_node" + log, err := logger.NewLogger(logPrm) fatalOnErr(err) @@ -581,14 +587,16 @@ func initCfg(appCfg *config.Config) *cfg { } func initInternals(appCfg *config.Config, log *logger.Logger) internals { + var healthStatus atomic.Int32 + healthStatus.Store(int32(control.HealthStatus_HEALTH_STATUS_UNDEFINED)) + return internals{ done: make(chan struct{}), appCfg: appCfg, internalErr: make(chan error), log: log, - wg: new(sync.WaitGroup), apiVersion: version.Current(), - healthStatus: atomic.NewInt32(int32(control.HealthStatus_HEALTH_STATUS_UNDEFINED)), + healthStatus: &healthStatus, } } @@ -614,7 +622,7 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt key: key, binPublicKey: key.PublicKey().Bytes(), localAddr: netAddr, - respSvc: response.NewService(response.WithNetworkState(netState)), + respSvc: response.NewService(netState), clientCache: cache.NewSDKClientCache(cacheOpts), bgClientCache: cache.NewSDKClientCache(cacheOpts), putClientCache: cache.NewSDKClientCache(cacheOpts), @@ -626,12 +634,14 @@ func initNetmap(appCfg *config.Config, netState *networkState, relayOnly bool) c netmapWorkerPool, err := ants.NewPool(notificationHandlerPoolSize) fatalOnErr(err) + var reBootstrapTurnedOff atomic.Bool + reBootstrapTurnedOff.Store(relayOnly) return cfgNetmap{ scriptHash: contractsconfig.Netmap(appCfg), state: netState, workerPool: netmapWorkerPool, needBootstrap: !relayOnly, - reBoostrapTurnedOff: atomic.NewBool(relayOnly), + reBoostrapTurnedOff: &reBootstrapTurnedOff, } } @@ -668,12 +678,12 @@ func (c *cfg) engineOpts() []engine.Option { opts = append(opts, engine.WithShardPoolSize(c.EngineCfg.shardPoolSize), engine.WithErrorThreshold(c.EngineCfg.errorThreshold), - engine.WithLogger(c.log), + engine.WithLowMemoryConsumption(c.EngineCfg.lowMem), ) if c.metricsCollector != nil { - opts = append(opts, engine.WithMetrics(c.metricsCollector)) + opts = append(opts, engine.WithMetrics(c.metricsCollector.Engine())) } return opts @@ -722,6 +732,9 @@ func (c *cfg) getPiloramaOpts(shCfg shardCfg) []pilorama.Option { pilorama.WithMaxBatchSize(prRead.maxBatchSize), pilorama.WithMaxBatchDelay(prRead.maxBatchDelay), ) + if c.metricsCollector != nil { + piloramaOpts = append(piloramaOpts, pilorama.WithMetrics(lsmetrics.NewPiloramaMetrics(c.metricsCollector.PiloramaMetrics()))) + } } return piloramaOpts } @@ -731,27 +744,46 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { for _, sRead := range shCfg.subStorages { switch sRead.typ { case blobovniczatree.Type: - ss = append(ss, blobstor.SubStorage{ - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(sRead.path), - blobovniczatree.WithPermissions(sRead.perm), - blobovniczatree.WithBlobovniczaSize(sRead.size), - blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), - blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), - blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobTreeOpts := []blobovniczatree.Option{ + blobovniczatree.WithRootPath(sRead.path), + blobovniczatree.WithPermissions(sRead.perm), + blobovniczatree.WithBlobovniczaSize(sRead.size), + blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), + blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), + blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobovniczatree.WithLogger(c.log), + } - blobovniczatree.WithLogger(c.log)), + if c.metricsCollector != nil { + blobTreeOpts = append(blobTreeOpts, + blobovniczatree.WithMetrics( + lsmetrics.NewBlobovniczaTreeMetrics(sRead.path, c.metricsCollector.BlobobvnizcaTreeMetrics()), + ), + ) + } + ss = append(ss, blobstor.SubStorage{ + Storage: blobovniczatree.NewBlobovniczaTree(blobTreeOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < shCfg.smallSizeObjectLimit }, }) case fstree.Type: + fstreeOpts := []fstree.Option{ + fstree.WithPath(sRead.path), + fstree.WithPerm(sRead.perm), + fstree.WithDepth(sRead.depth), + fstree.WithNoSync(sRead.noSync), + } + if c.metricsCollector != nil { + fstreeOpts = append(fstreeOpts, + fstree.WithMetrics( + lsmetrics.NewFSTreeMetricsWithoutShardID(sRead.path, c.metricsCollector.FSTree()), + ), + ) + } + ss = append(ss, blobstor.SubStorage{ - Storage: fstree.New( - fstree.WithPath(sRead.path), - fstree.WithPerm(sRead.perm), - fstree.WithDepth(sRead.depth), - fstree.WithNoSync(sRead.noSync)), + Storage: fstree.New(fstreeOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { return true }, @@ -769,31 +801,39 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { piloramaOpts := c.getPiloramaOpts(shCfg) ss := c.getSubstorageOpts(shCfg) + blobstoreOpts := []blobstor.Option{ + blobstor.WithCompressObjects(shCfg.compress), + blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), + blobstor.WithStorages(ss), + blobstor.WithLogger(c.log), + } + if c.metricsCollector != nil { + blobstoreOpts = append(blobstoreOpts, blobstor.WithMetrics(lsmetrics.NewBlobstoreMetrics(c.metricsCollector.Blobstore()))) + } + + mbOptions := []meta.Option{ + meta.WithPath(shCfg.metaCfg.path), + meta.WithPermissions(shCfg.metaCfg.perm), + meta.WithMaxBatchSize(shCfg.metaCfg.maxBatchSize), + meta.WithMaxBatchDelay(shCfg.metaCfg.maxBatchDelay), + meta.WithBoltDBOptions(&bbolt.Options{ + Timeout: 100 * time.Millisecond, + }), + meta.WithLogger(c.log), + meta.WithEpochState(c.cfgNetmap.state), + } + if c.metricsCollector != nil { + mbOptions = append(mbOptions, meta.WithMetrics(lsmetrics.NewMetabaseMetrics(shCfg.metaCfg.path, c.metricsCollector.MetabaseMetrics()))) + } + var sh shardOptsWithID sh.configID = shCfg.id() sh.shOpts = []shard.Option{ shard.WithLogger(c.log), shard.WithRefillMetabase(shCfg.refillMetabase), shard.WithMode(shCfg.mode), - shard.WithBlobStorOptions( - blobstor.WithCompressObjects(shCfg.compress), - blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), - blobstor.WithStorages(ss), - - blobstor.WithLogger(c.log), - ), - shard.WithMetaBaseOptions( - meta.WithPath(shCfg.metaCfg.path), - meta.WithPermissions(shCfg.metaCfg.perm), - meta.WithMaxBatchSize(shCfg.metaCfg.maxBatchSize), - meta.WithMaxBatchDelay(shCfg.metaCfg.maxBatchDelay), - meta.WithBoltDBOptions(&bbolt.Options{ - Timeout: 100 * time.Millisecond, - }), - - meta.WithLogger(c.log), - meta.WithEpochState(c.cfgNetmap.state), - ), + shard.WithBlobStorOptions(blobstoreOpts...), + shard.WithMetaBaseOptions(mbOptions...), shard.WithPiloramaOptions(piloramaOpts...), shard.WithWriteCache(shCfg.writecacheCfg.enabled), shard.WithWriteCacheOptions(writeCacheOpts...), @@ -842,18 +882,9 @@ func initLocalStorage(c *cfg) { // service will be created later c.cfgObject.getSvc = new(getsvc.Service) - var tssPrm tsourse.TombstoneSourcePrm - tssPrm.SetGetService(c.cfgObject.getSvc) - tombstoneSrc := tsourse.NewSource(tssPrm) - - tombstoneSource := tombstone.NewChecker( - tombstone.WithLogger(c.log), - tombstone.WithTombstoneSource(tombstoneSrc), - ) - var shardsAttached int for _, optsWithMeta := range c.shardOpts() { - id, err := ls.AddShard(append(optsWithMeta.shOpts, shard.WithTombstoneSource(tombstoneSource))...) + id, err := ls.AddShard(append(optsWithMeta.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))...) if err != nil { c.log.Error(logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err)) } else { @@ -966,13 +997,6 @@ func (c *cfg) needBootstrap() bool { return c.cfgNetmap.needBootstrap } -// ObjectServiceLoad implements system loader interface for policer component. -// It is calculated as size/capacity ratio of "remote object put" worker. -// Returns float value between 0.0 and 1.0. -func (c *cfg) ObjectServiceLoad() float64 { - return float64(c.cfgObject.pool.putRemote.Running()) / float64(c.cfgObject.pool.putRemoteCapacity) -} - type dCmp struct { name string reloadFunc func() error @@ -1031,6 +1055,10 @@ func (c *cfg) reloadConfig(ctx context.Context) { } components = append(components, dCmp{"logger", logPrm.Reload}) + components = append(components, dCmp{"runtime", func() error { + setRuntimeParameters(c) + return nil + }}) components = append(components, dCmp{"tracing", func() error { updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg)) if updated { @@ -1054,7 +1082,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { var rcfg engine.ReConfiguration for _, optsWithID := range c.shardOpts() { - rcfg.AddShard(optsWithID.configID, optsWithID.shOpts) + rcfg.AddShard(optsWithID.configID, append(optsWithID.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))) } err = c.cfgObject.cfgLocalStorage.localStorage.Reload(ctx, rcfg) @@ -1075,9 +1103,22 @@ func (c *cfg) reloadConfig(ctx context.Context) { c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } +func (c *cfg) createTombstoneSource() *tombstone.ExpirationChecker { + var tssPrm tsourse.TombstoneSourcePrm + tssPrm.SetGetService(c.cfgObject.getSvc) + tombstoneSrc := tsourse.NewSource(tssPrm) + + tombstoneSource := tombstone.NewChecker( + tombstone.WithLogger(c.log), + tombstone.WithTombstoneSource(tombstoneSrc), + ) + return tombstoneSource +} + func (c *cfg) shutdown() { c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN) + c.ctxCancel() c.done <- struct{}{} for i := range c.closers { c.closers[len(c.closers)-1-i].fn() diff --git a/cmd/frostfs-node/config/engine/config.go b/cmd/frostfs-node/config/engine/config.go index 36684f093..c944d1c58 100644 --- a/cmd/frostfs-node/config/engine/config.go +++ b/cmd/frostfs-node/config/engine/config.go @@ -83,3 +83,8 @@ func ShardPoolSize(c *config.Config) uint32 { func ShardErrorThreshold(c *config.Config) uint32 { return config.Uint32Safe(c.Sub(subsection), "shard_ro_error_threshold") } + +// EngineLowMemoryConsumption returns value of "lowmem" config parmeter from "storage" section. +func EngineLowMemoryConsumption(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "low_mem") +} diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/config.go index a6e34e80f..f8b2e2e9b 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/config.go @@ -5,8 +5,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/storage" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" ) // Config is a wrapper over the config section @@ -25,14 +23,11 @@ func (x *Config) Storages() []*storage.Config { typ := config.String( (*config.Config)(x), strconv.Itoa(i)+".type") - switch typ { - case "": + if typ == "" { return ss - case fstree.Type, blobovniczatree.Type: - sub := storage.From((*config.Config)(x).Sub(strconv.Itoa(i))) - ss = append(ss, sub) - default: - panic("invalid type") } + + sub := storage.From((*config.Config)(x).Sub(strconv.Itoa(i))) + ss = append(ss, sub) } } diff --git a/cmd/frostfs-node/config/policer/config.go b/cmd/frostfs-node/config/policer/config.go index 51e55c341..487e42be8 100644 --- a/cmd/frostfs-node/config/policer/config.go +++ b/cmd/frostfs-node/config/policer/config.go @@ -25,3 +25,9 @@ func HeadTimeout(c *config.Config) time.Duration { return HeadTimeoutDefault } + +// UnsafeDisable returns the value of "unsafe_disable" config parameter +// from "policer" section. +func UnsafeDisable(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "unsafe_disable") +} diff --git a/cmd/frostfs-node/config/profiler/config.go b/cmd/frostfs-node/config/profiler/config.go index f891833a6..191694970 100644 --- a/cmd/frostfs-node/config/profiler/config.go +++ b/cmd/frostfs-node/config/profiler/config.go @@ -51,3 +51,27 @@ func Address(c *config.Config) string { return AddressDefault } + +// BlockRates returns the value of "block_rate" config parameter +// from "pprof" section. +func BlockRate(c *config.Config) int { + s := c.Sub(subsection) + + v := int(config.IntSafe(s, "block_rate")) + if v <= 0 { + return 0 + } + return v +} + +// MutexRate returns the value of "mutex_rate" config parameter +// from "pprof" section. +func MutexRate(c *config.Config) int { + s := c.Sub(subsection) + + v := int(config.IntSafe(s, "mutex_rate")) + if v <= 0 { + return 0 + } + return v +} diff --git a/cmd/frostfs-node/config/profiler/config_test.go b/cmd/frostfs-node/config/profiler/config_test.go index bb1b20eb8..355874387 100644 --- a/cmd/frostfs-node/config/profiler/config_test.go +++ b/cmd/frostfs-node/config/profiler/config_test.go @@ -18,6 +18,9 @@ func TestProfilerSection(t *testing.T) { require.Equal(t, profilerconfig.ShutdownTimeoutDefault, to) require.Equal(t, profilerconfig.AddressDefault, addr) require.False(t, profilerconfig.Enabled(configtest.EmptyConfig())) + + require.Zero(t, profilerconfig.BlockRate(configtest.EmptyConfig())) + require.Zero(t, profilerconfig.MutexRate(configtest.EmptyConfig())) }) const path = "../../../../config/example/node" @@ -29,6 +32,9 @@ func TestProfilerSection(t *testing.T) { require.Equal(t, 15*time.Second, to) require.Equal(t, "localhost:6060", addr) require.True(t, profilerconfig.Enabled(c)) + + require.Equal(t, 10_000, profilerconfig.BlockRate(c)) + require.Equal(t, 10_000, profilerconfig.MutexRate(c)) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/config/runtime/config.go b/cmd/frostfs-node/config/runtime/config.go new file mode 100644 index 000000000..ad6cce43b --- /dev/null +++ b/cmd/frostfs-node/config/runtime/config.go @@ -0,0 +1,23 @@ +package runtime + +import ( + "math" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" +) + +const ( + subsection = "runtime" + memoryLimitDefault = math.MaxInt64 +) + +// GCMemoryLimitBytes returns the value of "soft_memory_limit" config parameter from "runtime" section. +func GCMemoryLimitBytes(c *config.Config) int64 { + l := config.SizeInBytesSafe(c.Sub(subsection), "soft_memory_limit") + + if l > 0 { + return int64(l) + } + + return memoryLimitDefault +} diff --git a/cmd/frostfs-node/config/runtime/config_test.go b/cmd/frostfs-node/config/runtime/config_test.go new file mode 100644 index 000000000..1bfa42ad8 --- /dev/null +++ b/cmd/frostfs-node/config/runtime/config_test.go @@ -0,0 +1,30 @@ +package runtime + +import ( + "math" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "github.com/stretchr/testify/require" +) + +func TestGCMemoryLimit(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + empty := configtest.EmptyConfig() + + require.Equal(t, int64(math.MaxInt64), GCMemoryLimitBytes(empty)) + }) + + const path = "../../../../config/example/node" + + fileConfigTest := func(c *config.Config) { + require.Equal(t, int64(1073741824), GCMemoryLimitBytes(c)) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) +} diff --git a/cmd/frostfs-node/config/tracing/config.go b/cmd/frostfs-node/config/tracing/config.go index 76572cc31..e846be158 100644 --- a/cmd/frostfs-node/config/tracing/config.go +++ b/cmd/frostfs-node/config/tracing/config.go @@ -1,9 +1,9 @@ package tracing import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" ) const ( diff --git a/cmd/frostfs-node/config/tree/config.go b/cmd/frostfs-node/config/tree/config.go index f6087c53d..8a8919999 100644 --- a/cmd/frostfs-node/config/tree/config.go +++ b/cmd/frostfs-node/config/tree/config.go @@ -1,9 +1,11 @@ package treeconfig import ( + "fmt" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) const ( @@ -71,3 +73,22 @@ func (c TreeConfig) ReplicationWorkerCount() int { func (c TreeConfig) SyncInterval() time.Duration { return config.DurationSafe(c.cfg, "sync_interval") } + +// AuthorizedKeys parses and returns an array of "authorized_keys" config +// parameter from "tree" section. +// +// Returns an empty list if not set. +func (c TreeConfig) AuthorizedKeys() keys.PublicKeys { + authorizedKeysStr := config.StringSliceSafe(c.cfg, "authorized_keys") + authorizedKeys := make(keys.PublicKeys, 0, len(authorizedKeysStr)) + + for i := range authorizedKeysStr { + pub, err := keys.NewPublicKeyFromString(authorizedKeysStr[i]) + if err != nil { + panic(fmt.Errorf("could not parse Tree authorized key %s: %w", authorizedKeysStr[i], err)) + } + + authorizedKeys = append(authorizedKeys, pub) + } + return authorizedKeys +} diff --git a/cmd/frostfs-node/config/tree/config_test.go b/cmd/frostfs-node/config/tree/config_test.go index a39aa4553..898f7e715 100644 --- a/cmd/frostfs-node/config/tree/config_test.go +++ b/cmd/frostfs-node/config/tree/config_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" ) @@ -21,10 +22,19 @@ func TestTreeSection(t *testing.T) { require.Equal(t, 0, treeSec.ReplicationChannelCapacity()) require.Equal(t, 0, treeSec.ReplicationWorkerCount()) require.Equal(t, time.Duration(0), treeSec.ReplicationTimeout()) + require.Equal(t, 0, len(treeSec.AuthorizedKeys())) }) const path = "../../../../config/example/node" + var expectedKeys keys.PublicKeys + key, err := keys.NewPublicKeyFromString("0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0") + require.NoError(t, err) + expectedKeys = append(expectedKeys, key) + key, err = keys.NewPublicKeyFromString("02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56") + require.NoError(t, err) + expectedKeys = append(expectedKeys, key) + var fileConfigTest = func(c *config.Config) { treeSec := treeconfig.Tree(c) @@ -34,6 +44,7 @@ func TestTreeSection(t *testing.T) { require.Equal(t, 32, treeSec.ReplicationWorkerCount()) require.Equal(t, 5*time.Second, treeSec.ReplicationTimeout()) require.Equal(t, time.Hour, treeSec.SyncInterval()) + require.Equal(t, expectedKeys, treeSec.AuthorizedKeys()) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 569e4a7ca..9a6cfd02b 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -83,16 +83,13 @@ func initContainerService(ctx context.Context, c *cfg) { server := containerTransportGRPC.New( containerService.NewSignService( &c.key.PrivateKey, - containerService.NewResponseService( - &usedSpaceService{ - Server: containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt)), - loadWriterProvider: loadRouter, - loadPlacementBuilder: loadPlacementBuilder, - routeBuilder: routeBuilder, - cfg: c, - }, - c.respSvc, - ), + &usedSpaceService{ + Server: containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), + loadWriterProvider: loadRouter, + loadPlacementBuilder: loadPlacementBuilder, + routeBuilder: routeBuilder, + cfg: c, + }, ), ) @@ -452,7 +449,7 @@ type localStorageLoad struct { } func (d *localStorageLoad) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontroller.UsedSpaceHandler) error { - idList, err := engine.ListContainers(d.engine) + idList, err := engine.ListContainers(context.TODO(), d.engine) if err != nil { return fmt.Errorf("list containers on engine failure: %w", err) } @@ -575,6 +572,8 @@ func (c *usedSpaceService) AnnounceUsedSpace(ctx context.Context, req *container resp := new(containerV2.AnnounceUsedSpaceResponse) resp.SetBody(respBody) + c.cfg.respSvc.SetMeta(resp) + return resp, nil } diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index f4b068419..3ed6bc54a 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -80,7 +80,7 @@ func (c *cfg) setHealthStatus(st control.HealthStatus) { c.healthStatus.Store(int32(st)) if c.metricsCollector != nil { - c.metricsCollector.SetHealth(int32(st)) + c.metricsCollector.State().SetHealth(int32(st)) } } diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index b0a587782..1dd0f0729 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -7,25 +7,30 @@ import ( "net" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) +const maxRecvMsgSize = 256 << 20 + func initGRPC(c *cfg) { var successCount int grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { serverOpts := []grpc.ServerOption{ - grpc.MaxSendMsgSize(maxMsgSize), + grpc.MaxRecvMsgSize(maxRecvMsgSize), grpc.ChainUnaryInterceptor( - tracing.NewGRPCUnaryServerInterceptor(), + metrics.NewUnaryServerInterceptor(), + tracing.NewUnaryServerInterceptor(), ), grpc.ChainStreamInterceptor( - tracing.NewGRPCStreamServerInterceptor(), + metrics.NewStreamServerInterceptor(), + tracing.NewStreamServerInterceptor(), ), } diff --git a/cmd/frostfs-node/keyspaceiterator.go b/cmd/frostfs-node/keyspaceiterator.go new file mode 100644 index 000000000..e7214aacb --- /dev/null +++ b/cmd/frostfs-node/keyspaceiterator.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + "fmt" + + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" +) + +type keySpaceIterator struct { + ng *engine.StorageEngine + cur *engine.Cursor +} + +func (it *keySpaceIterator) Next(ctx context.Context, batchSize uint32) ([]objectcore.AddressWithType, error) { + var prm engine.ListWithCursorPrm + prm.WithCursor(it.cur) + prm.WithCount(batchSize) + + res, err := it.ng.ListWithCursor(ctx, prm) + if err != nil { + return nil, fmt.Errorf("cannot list objects in engine: %w", err) + } + + it.cur = res.Cursor() + return res.AddressList(), nil +} + +func (it *keySpaceIterator) Rewind() { + it.cur = nil +} diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 4fcc38e08..bf872da03 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -57,7 +57,8 @@ func main() { c := initCfg(appCfg) - ctx, cancel := context.WithCancel(context.Background()) + var ctx context.Context + ctx, c.ctxCancel = context.WithCancel(context.Background()) initApp(ctx, c) @@ -67,7 +68,7 @@ func main() { c.setHealthStatus(control.HealthStatus_READY) - wait(c, cancel) + wait(c) } func initAndLog(c *cfg, name string, initializer func(*cfg)) { @@ -83,9 +84,9 @@ func initApp(ctx context.Context, c *cfg) { c.wg.Done() }() - pprof, _ := pprofComponent(c) + setRuntimeParameters(c) metrics, _ := metricsComponent(c) - initAndLog(c, pprof.name, pprof.init) + initAndLog(c, "profiler", initProfilerService) initAndLog(c, metrics.name, metrics.init) initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) }) @@ -141,14 +142,12 @@ func bootUp(ctx context.Context, c *cfg) { startWorkers(ctx, c) } -func wait(c *cfg, cancel func()) { +func wait(c *cfg) { c.log.Info(logs.CommonApplicationStarted, zap.String("version", misc.Version)) <-c.done // graceful shutdown - cancel() - c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) c.wg.Wait() diff --git a/cmd/frostfs-node/metrics.go b/cmd/frostfs-node/metrics.go index cf621086d..19b4af51f 100644 --- a/cmd/frostfs-node/metrics.go +++ b/cmd/frostfs-node/metrics.go @@ -2,7 +2,7 @@ package main import ( metricsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/metrics" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) func metricsComponent(c *cfg) (*httpComponent, bool) { diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 2e086f994..63d1605ef 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -9,6 +9,7 @@ import ( morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -29,8 +30,6 @@ const ( ) func initMorphComponents(ctx context.Context, c *cfg) { - var err error - addresses := morphconfig.RPCEndpoint(c.appCfg) // Morph client stable-sorts endpoints by priority. Shuffle here to randomize @@ -43,11 +42,13 @@ func initMorphComponents(ctx context.Context, c *cfg) { c.key, client.WithDialTimeout(morphconfig.DialTimeout(c.appCfg)), client.WithLogger(c.log), + client.WithMetrics(metrics.NewMorphClientMetrics()), client.WithEndpoints(addresses...), client.WithConnLostCallback(func() { c.internalErr <- errors.New("morph connection has been lost") }), client.WithSwitchInterval(morphconfig.SwitchInterval(c.appCfg)), + client.WithMorphCacheMetrics(metrics.NewNodeMorphCacheMetrics()), ) if err != nil { c.log.Info(logs.FrostFSNodeFailedToCreateNeoRPCClient, diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 58e3cb2f2..96b866b11 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "sync/atomic" netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -19,7 +20,6 @@ import ( netmapService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/netmap" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -27,7 +27,7 @@ import ( type networkState struct { epoch *atomic.Uint64 - controlNetStatus atomic.Value // control.NetmapStatus + controlNetStatus atomic.Int32 // control.NetmapStatus nodeInfo atomic.Value // *netmapSDK.NodeInfo @@ -35,13 +35,11 @@ type networkState struct { } func newNetworkState() *networkState { - var nmStatus atomic.Value - nmStatus.Store(control.NetmapStatus_STATUS_UNDEFINED) - - return &networkState{ - epoch: atomic.NewUint64(0), - controlNetStatus: nmStatus, + ns := &networkState{ + epoch: new(atomic.Uint64), } + ns.controlNetStatus.Store(int32(control.NetmapStatus_STATUS_UNDEFINED)) + return ns } func (s *networkState) CurrentEpoch() uint64 { @@ -91,11 +89,11 @@ func (s *networkState) setNodeInfo(ni *netmapSDK.NodeInfo) { // calls will process this value to decide what status node should set in the // network. func (s *networkState) setControlNetmapStatus(st control.NetmapStatus) { - s.controlNetStatus.Store(st) + s.controlNetStatus.Store(int32(st)) } func (s *networkState) controlNetmapStatus() (res control.NetmapStatus) { - return s.controlNetStatus.Load().(control.NetmapStatus) + return control.NetmapStatus(s.controlNetStatus.Load()) } func (s *networkState) getNodeInfo() (res netmapSDK.NodeInfo, ok bool) { @@ -150,17 +148,15 @@ func initNetmapService(ctx context.Context, c *cfg) { server := netmapTransportGRPC.New( netmapService.NewSignService( &c.key.PrivateKey, - netmapService.NewResponseService( - netmapService.NewExecutionService( - c, - c.apiVersion, - &netInfo{ - netState: c.cfgNetmap.state, - magic: c.cfgMorph.client, - morphClientNetMap: c.cfgNetmap.wrapper, - msPerBlockRdr: c.cfgMorph.client.MsPerBlock, - }, - ), + netmapService.NewExecutionService( + c, + c.apiVersion, + &netInfo{ + netState: c.cfgNetmap.state, + magic: c.cfgMorph.client, + morphClientNetMap: c.cfgNetmap.wrapper, + msPerBlockRdr: c.cfgMorph.client.MsPerBlock, + }, c.respSvc, ), ), @@ -227,6 +223,14 @@ func addNewEpochNotificationHandlers(c *cfg) { // Must be called after initNetmapService. func bootstrapNode(c *cfg) { if c.needBootstrap() { + if c.IsMaintenance() { + c.log.Info(logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) + return + } + if c.isOnlineCandidate { + c.log.Info(logs.NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap) + return + } err := c.bootstrap() fatalOnErrDetails("bootstrap error", err) } @@ -258,30 +262,77 @@ func initNetmapState(c *cfg) { epoch, err := c.cfgNetmap.wrapper.Epoch() fatalOnErrDetails("could not initialize current epoch number", err) - ni, err := c.netmapLocalNodeState(epoch) + var ni *netmapSDK.NodeInfo + ni, c.isOnlineCandidate, err = c.netmapInitLocalNodeState(epoch) fatalOnErrDetails("could not init network state", err) - stateWord := "undefined" - - if ni != nil { - switch { - case ni.IsOnline(): - stateWord = "online" - case ni.IsOffline(): - stateWord = "offline" - } - } + stateWord := nodeState(ni) c.log.Info(logs.FrostFSNodeInitialNetworkState, zap.Uint64("epoch", epoch), zap.String("state", stateWord), ) + if ni != nil && ni.IsMaintenance() { + c.isMaintenance.Store(true) + } + c.cfgNetmap.state.setCurrentEpoch(epoch) c.cfgNetmap.startEpoch = epoch c.handleLocalNodeInfo(ni) } +func nodeState(ni *netmapSDK.NodeInfo) string { + if ni != nil { + switch { + case ni.IsOnline(): + return "online" + case ni.IsOffline(): + return "offline" + case ni.IsMaintenance(): + return "maintenance" + } + } + return "undefined" +} + +func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, bool, error) { + nmNodes, err := c.cfgNetmap.wrapper.GetCandidates() + if err != nil { + return nil, false, err + } + + var candidate *netmapSDK.NodeInfo + isOnlineCandidate := false + for i := range nmNodes { + if bytes.Equal(nmNodes[i].PublicKey(), c.binPublicKey) { + candidate = &nmNodes[i] + isOnlineCandidate = candidate.IsOnline() + break + } + } + + node, err := c.netmapLocalNodeState(epoch) + if err != nil { + return nil, false, err + } + + if candidate == nil { + return node, false, nil + } + + nmState := nodeState(node) + candidateState := nodeState(candidate) + if nmState != candidateState { + // This happens when the node was switched to maintenance without epoch tick. + // We expect it to continue staying in maintenance. + c.log.Info("candidate status is different from the netmap status, the former takes priority", + zap.String("netmap", nmState), + zap.String("candidate", candidateState)) + } + return candidate, isOnlineCandidate, nil +} + func (c *cfg) netmapLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { // calculate current network state nm, err := c.cfgNetmap.wrapper.GetNetMapByEpoch(epoch) diff --git a/cmd/frostfs-node/notificator.go b/cmd/frostfs-node/notificator.go index 358b39a72..3fa486955 100644 --- a/cmd/frostfs-node/notificator.go +++ b/cmd/frostfs-node/notificator.go @@ -27,7 +27,7 @@ type notificationSource struct { func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler func(topic string, addr oid.Address)) { log := n.l.With(zap.Uint64("epoch", epoch)) - listRes, err := n.e.ListContainers(engine.ListContainersPrm{}) + listRes, err := n.e.ListContainers(ctx, engine.ListContainersPrm{}) if err != nil { log.Error(logs.FrostFSNodeNotificatorCouldNotListContainers, zap.Error(err)) return diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 4ff9b8522..84411d31b 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -38,6 +38,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -69,6 +70,10 @@ func (s *objectSvc) Put() (objectService.PutObjectStream, error) { return s.put.Put() } +func (s *objectSvc) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + return s.put.PutSingle(ctx, req) +} + func (s *objectSvc) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { return s.get.Head(ctx, req) } @@ -195,7 +200,7 @@ func initObjectService(c *cfg) { ) c.shared.metricsSvc = objectService.NewMetricCollector( - signSvc, c.metricsCollector, metricsconfig.Enabled(c.appCfg)) + signSvc, c.metricsCollector.ObjectService(), metricsconfig.Enabled(c.appCfg)) server := objectTransportGRPC.New(c.shared.metricsSvc) for _, srv := range c.cfgGRPC.servers { @@ -204,17 +209,37 @@ func initObjectService(c *cfg) { } func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.ClientCache) { + if policerconfig.UnsafeDisable(c.appCfg) { + c.log.Warn(logs.FrostFSNodePolicerIsDisabled) + return + } + ls := c.cfgObject.cfgLocalStorage.localStorage + buryFn := func(ctx context.Context, addr oid.Address) error { + var prm engine.InhumePrm + prm.MarkAsGarbage(addr) + prm.WithForceRemoval() + + _, err := ls.Inhume(ctx, prm) + return err + } + + remoteHeader := headsvc.NewRemoteHeader(keyStorage, clientConstructor) + pol := policer.New( policer.WithLogger(c.log), - policer.WithLocalStorage(ls), + policer.WithKeySpaceIterator(&keySpaceIterator{ng: ls}), + policer.WithBuryFunc(buryFn), policer.WithContainerSource(c.cfgObject.cnrSource), policer.WithPlacementBuilder( placement.NewNetworkMapSourceBuilder(c.netMapSource), ), - policer.WithRemoteHeader( - headsvc.NewRemoteHeader(keyStorage, clientConstructor), + policer.WithRemoteObjectHeaderFunc( + func(ctx context.Context, ni netmapSDK.NodeInfo, a oid.Address) (*objectSDK.Object, error) { + prm := new(headsvc.RemoteHeadPrm).WithNodeInfo(ni).WithObjectAddress(a) + return remoteHeader.Head(ctx, prm) + }, ), policer.WithNetmapKeys(c), policer.WithHeadTimeout( @@ -234,7 +259,6 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl }), policer.WithMaxCapacity(c.cfgObject.pool.replicatorPoolSize), policer.WithPool(c.cfgObject.pool.replication), - policer.WithNodeLoader(c), ) c.workers = append(c.workers, worker{ @@ -267,6 +291,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa replicator.WithRemoteSender( putsvc.NewRemoteSender(keyStorage, cache), ), + replicator.WithMetrics(c.metricsCollector.Replicator()), ) } @@ -287,48 +312,40 @@ func createPutSvc(c *cfg, keyStorage *util.KeyStorage) *putsvc.Service { } return putsvc.NewService( - putsvc.WithKeyStorage(keyStorage), - putsvc.WithClientConstructor(c.putClientCache), - putsvc.WithMaxSizeSource(newCachedMaxObjectSizeSource(c)), - putsvc.WithObjectStorage(os), - putsvc.WithContainerSource(c.cfgObject.cnrSource), - putsvc.WithNetworkMapSource(c.netMapSource), - putsvc.WithNetmapKeys(c), - putsvc.WithNetworkState(c.cfgNetmap.state), + keyStorage, + c.putClientCache, + newCachedMaxObjectSizeSource(c), + os, + c.cfgObject.cnrSource, + c.netMapSource, + c, + c.cfgNetmap.state, putsvc.WithWorkerPools(c.cfgObject.pool.putRemote, c.cfgObject.pool.putLocal), putsvc.WithLogger(c.log), ) } func createPutSvcV2(sPut *putsvc.Service, keyStorage *util.KeyStorage) *putsvcV2.Service { - return putsvcV2.NewService( - putsvcV2.WithInternalService(sPut), - putsvcV2.WithKeyStorage(keyStorage), - ) + return putsvcV2.NewService(sPut, keyStorage) } func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache) *searchsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage return searchsvc.New( - searchsvc.WithLogger(c.log), - searchsvc.WithLocalStorageEngine(ls), - searchsvc.WithClientConstructor(coreConstructor), - searchsvc.WithTraverserGenerator( - traverseGen.WithTraverseOptions( - placement.WithoutSuccessTracking(), - ), + ls, + coreConstructor, + traverseGen.WithTraverseOptions( + placement.WithoutSuccessTracking(), ), - searchsvc.WithNetMapSource(c.netMapSource), - searchsvc.WithKeyStorage(keyStorage), + c.netMapSource, + keyStorage, + searchsvc.WithLogger(c.log), ) } func createSearchSvcV2(sSearch *searchsvc.Service, keyStorage *util.KeyStorage) *searchsvcV2.Service { - return searchsvcV2.NewService( - searchsvcV2.WithInternalService(sSearch), - searchsvcV2.WithKeyStorage(keyStorage), - ) + return searchsvcV2.NewService(sSearch, keyStorage) } func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, @@ -356,24 +373,22 @@ func createGetServiceV2(sGet *getsvc.Service, keyStorage *util.KeyStorage) *gets func createDeleteService(c *cfg, keyStorage *util.KeyStorage, sGet *getsvc.Service, sSearch *searchsvc.Service, sPut *putsvc.Service) *deletesvc.Service { return deletesvc.New( - deletesvc.WithLogger(c.log), - deletesvc.WithHeadService(sGet), - deletesvc.WithSearchService(sSearch), - deletesvc.WithPutService(sPut), - deletesvc.WithNetworkInfo(&delNetInfo{ + sGet, + sSearch, + sPut, + &delNetInfo{ State: c.cfgNetmap.state, tsLifetime: c.cfgObject.tombstoneLifetime, cfg: c, - }), - deletesvc.WithKeyStorage(keyStorage), + }, + keyStorage, + deletesvc.WithLogger(c.log), ) } func createDeleteServiceV2(sDelete *deletesvc.Service) *deletesvcV2.Service { - return deletesvcV2.NewService( - deletesvcV2.WithInternalService(sDelete), - ) + return deletesvcV2.NewService(sDelete) } func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Service, @@ -395,21 +410,16 @@ func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter) v2.Se irFetcher := createInnerRingFetcher(c) return v2.New( + splitSvc, + c.netMapSource, + newCachedIRFetcher(irFetcher), + acl.NewChecker( + c.cfgNetmap.state, + c.cfgObject.eaclSource, + eaclSDK.NewValidator(), + ls), + c.cfgObject.cnrSource, v2.WithLogger(c.log), - v2.WithIRFetcher(newCachedIRFetcher(irFetcher)), - v2.WithNetmapSource(c.netMapSource), - v2.WithContainerSource( - c.cfgObject.cnrSource, - ), - v2.WithNextService(splitSvc), - v2.WithEACLChecker( - acl.NewChecker(new(acl.CheckerPrm). - SetNetmapState(c.cfgNetmap.state). - SetEACLSource(c.cfgObject.eaclSource). - SetValidator(eaclSDK.NewValidator()). - SetLocalStorage(ls), - ), - ), ) } @@ -429,7 +439,7 @@ func (s *morphEACLFetcher) GetEACL(cnr cid.ID) (*containercore.EACL, error) { } if !eaclInfo.Signature.Verify(binTable) { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return nil, errors.New("invalid signature of the eACL table") } diff --git a/cmd/frostfs-node/pprof.go b/cmd/frostfs-node/pprof.go index 9be2dd9df..dcd320146 100644 --- a/cmd/frostfs-node/pprof.go +++ b/cmd/frostfs-node/pprof.go @@ -1,10 +1,19 @@ package main import ( + "runtime" + profilerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/profiler" httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" ) +func initProfilerService(c *cfg) { + tuneProfilers(c) + + pprof, _ := pprofComponent(c) + pprof.init(c) +} + func pprofComponent(c *cfg) (*httpComponent, bool) { var updated bool // check if it has been inited before @@ -13,6 +22,7 @@ func pprofComponent(c *cfg) (*httpComponent, bool) { c.dynamicConfiguration.pprof.cfg = c c.dynamicConfiguration.pprof.name = "pprof" c.dynamicConfiguration.pprof.handler = httputil.Handler() + c.dynamicConfiguration.pprof.preReload = tuneProfilers updated = true } @@ -35,3 +45,18 @@ func pprofComponent(c *cfg) (*httpComponent, bool) { return c.dynamicConfiguration.pprof, updated } + +func tuneProfilers(c *cfg) { + // Disabled by default, see documentation for + // runtime.SetBlockProfileRate() and runtime.SetMutexProfileFraction(). + blockRate := 0 + mutexRate := 0 + + if profilerconfig.Enabled(c.appCfg) { + blockRate = profilerconfig.BlockRate(c.appCfg) + mutexRate = profilerconfig.MutexRate(c.appCfg) + } + + runtime.SetBlockProfileRate(blockRate) + runtime.SetMutexProfileFraction(mutexRate) +} diff --git a/cmd/frostfs-node/runtime.go b/cmd/frostfs-node/runtime.go new file mode 100644 index 000000000..d858ba490 --- /dev/null +++ b/cmd/frostfs-node/runtime.go @@ -0,0 +1,26 @@ +package main + +import ( + "os" + "runtime/debug" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/runtime" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "go.uber.org/zap" +) + +func setRuntimeParameters(c *cfg) { + if len(os.Getenv("GOMEMLIMIT")) != 0 { + // default limit < yaml limit < app env limit < GOMEMLIMIT + c.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) + return + } + + memLimitBytes := runtime.GCMemoryLimitBytes(c.appCfg) + previous := debug.SetMemoryLimit(memLimitBytes) + if memLimitBytes != previous { + c.log.Info(logs.RuntimeSoftMemoryLimitUpdated, + zap.Int64("new_value", memLimitBytes), + zap.Int64("old_value", previous)) + } +} diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index 95e3b8205..f9c1811a1 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -53,10 +53,7 @@ func initSessionService(c *cfg) { server := sessionTransportGRPC.New( sessionSvc.NewSignService( &c.key.PrivateKey, - sessionSvc.NewResponseService( - sessionSvc.NewExecutionService(c.privateTokenStore, c.log), - c.respSvc, - ), + sessionSvc.NewExecutionService(c.privateTokenStore, c.respSvc, c.log), ), ) diff --git a/cmd/frostfs-node/tracing.go b/cmd/frostfs-node/tracing.go index d963ba866..08dc049da 100644 --- a/cmd/frostfs-node/tracing.go +++ b/cmd/frostfs-node/tracing.go @@ -4,9 +4,9 @@ import ( "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.uber.org/zap" ) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index b4f43acac..1f73b7956 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -55,7 +55,9 @@ func initTreeService(c *cfg) { tree.WithContainerCacheSize(treeConfig.CacheSize()), tree.WithReplicationTimeout(treeConfig.ReplicationTimeout()), tree.WithReplicationChannelCapacity(treeConfig.ReplicationChannelCapacity()), - tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount())) + tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount()), + tree.WithAuthorizedKeys(treeConfig.AuthorizedKeys()), + tree.WithMetrics(c.metricsCollector.TreeService())) for _, srv := range c.cfgGRPC.servers { tree.RegisterTreeServiceServer(srv, c.treeService) diff --git a/cmd/frostfs-node/validate.go b/cmd/frostfs-node/validate.go index 3896fd6be..e07afb2ca 100644 --- a/cmd/frostfs-node/validate.go +++ b/cmd/frostfs-node/validate.go @@ -51,17 +51,13 @@ func validateConfig(c *config.Config) error { blobstor := sc.BlobStor().Storages() if len(blobstor) != 2 { - // TODO (@fyrcik): remove after #1522 return fmt.Errorf("blobstor section must have 2 components, got: %d", len(blobstor)) } for i := range blobstor { switch blobstor[i].Type() { case fstree.Type, blobovniczatree.Type: default: - // FIXME #1764 (@fyrchik): this line is currently unreachable, - // because we panic in `sc.BlobStor().Storages()`. - return fmt.Errorf("unexpected storage type: %s (shard %d)", - blobstor[i].Type(), shardNum) + return fmt.Errorf("unexpected storage type: %s (shard %d)", blobstor[i].Type(), shardNum) } if blobstor[i].Perm()&0600 != 0600 { return fmt.Errorf("invalid permissions for blobstor component: %s, "+ diff --git a/cmd/internal/common/exit.go b/cmd/internal/common/exit.go index 9b912ddc0..9e4fa3098 100644 --- a/cmd/internal/common/exit.go +++ b/cmd/internal/common/exit.go @@ -46,5 +46,8 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) { } cmd.PrintErrln(err) + if cmd.PersistentPostRun != nil { + cmd.PersistentPostRun(cmd, nil) + } os.Exit(code) } diff --git a/config/example/ir.env b/config/example/ir.env index e3de23ac1..3f9530ab6 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -28,7 +28,6 @@ FROSTFS_IR_LOCODE_DB_PATH=/path/to/locode.db FROSTFS_IR_FEE_MAIN_CHAIN=50000000 FROSTFS_IR_FEE_SIDE_CHAIN=200000000 -FROSTFS_IR_FEE_NAMED_CONTAINER_REGISTER=2500000000 FROSTFS_IR_TIMERS_EMIT=240 FROSTFS_IR_TIMERS_STOP_ESTIMATION_MUL=1 @@ -74,6 +73,8 @@ FROSTFS_IR_CONTRACTS_ALPHABET_ZHIVETE=f584699bc2ff457d339fb09f16217042c1a42101 FROSTFS_IR_PPROF_ENABLED=true FROSTFS_IR_PPROF_ADDRESS=localhost:6060 FROSTFS_IR_PPROF_SHUTDOWN_TIMEOUT=30s +FROSTFS_IR_PPROF_BLOCK_RATE=10000 +FROSTFS_IR_PPROF_MUTEX_RATE=10000 FROSTFS_IR_PROMETHEUS_ENABLED=true FROSTFS_IR_PROMETHEUS_ADDRESS=localhost:9090 diff --git a/config/example/ir.yaml b/config/example/ir.yaml index bd56ec74b..a01f3d0bb 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -49,7 +49,6 @@ locode: fee: main_chain: 50000000 # Fixed8 value of extra GAS fee for mainchain contract invocation; ignore if notary is enabled in mainchain side_chain: 200000000 # Fixed8 value of extra GAS fee for sidechain contract invocation; ignore if notary is enabled in sidechain - named_container_register: 2500000000 # Fixed8 value of extra GAS fee for named conatiner registration in container contract; ignore if notary is enabled in sidechain timers: emit: 240 # Number of sidechain blocks between GAS emission cycles; disabled by default @@ -113,6 +112,8 @@ pprof: enabled: true address: localhost:6060 # Endpoint for application pprof profiling; disabled by default shutdown_timeout: 30s # Timeout for profiling HTTP server graceful shutdown + block_rate: 10000 # sampling rate: an average of one blocking event per rate nanoseconds spent blocked is reported; "1" reports every blocking event; "0" disables profiler + mutex_rate: 10000 # sampling rate: on average 1/rate events are reported; "0" disables profiler prometheus: enabled: true diff --git a/config/example/node.env b/config/example/node.env index 77992d995..3abb744be 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -3,6 +3,8 @@ FROSTFS_LOGGER_LEVEL=debug FROSTFS_PPROF_ENABLED=true FROSTFS_PPROF_ADDRESS=localhost:6060 FROSTFS_PPROF_SHUTDOWN_TIMEOUT=15s +FROSTFS_PPROF_BLOCK_RATE=10000 +FROSTFS_PPROF_MUTEX_RATE=10000 FROSTFS_PROMETHEUS_ENABLED=true FROSTFS_PROMETHEUS_ADDRESS=localhost:9090 @@ -34,6 +36,7 @@ FROSTFS_TREE_REPLICATION_CHANNEL_CAPACITY=32 FROSTFS_TREE_REPLICATION_WORKER_COUNT=32 FROSTFS_TREE_REPLICATION_TIMEOUT=5s FROSTFS_TREE_SYNC_INTERVAL=1h +FROSTFS_TREE_AUTHORIZED_KEYS="0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0 02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56" # gRPC section ## 0 server @@ -185,3 +188,5 @@ FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m FROSTFS_TRACING_ENABLED=true FROSTFS_TRACING_ENDPOINT="localhost" FROSTFS_TRACING_EXPORTER="otlp_grpc" + +FROSTFS_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/example/node.json b/config/example/node.json index b52eb6d96..6c98903f1 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -5,7 +5,9 @@ "pprof": { "enabled": true, "address": "localhost:6060", - "shutdown_timeout": "15s" + "shutdown_timeout": "15s", + "block_rate": 10000, + "mutex_rate": 10000 }, "prometheus": { "enabled": true, @@ -73,7 +75,11 @@ "replication_channel_capacity": 32, "replication_worker_count": 32, "replication_timeout": "5s", - "sync_interval": "1h" + "sync_interval": "1h", + "authorized_keys": [ + "0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0", + "02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56" + ] }, "control": { "authorized_keys": [ @@ -239,5 +245,8 @@ "enabled": true, "endpoint": "localhost:9090", "exporter": "otlp_grpc" + }, + "runtime": { + "soft_memory_limit": 1073741824 } } diff --git a/config/example/node.yaml b/config/example/node.yaml index 1669e0e86..0ef5fea7f 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -5,6 +5,8 @@ pprof: enabled: true address: localhost:6060 # endpoint for Node profiling shutdown_timeout: 15s # timeout for profiling HTTP server graceful shutdown + block_rate: 10000 # sampling rate: an average of one blocking event per rate nanoseconds spent blocked is reported; "1" reports every blocking event; "0" disables profiler + mutex_rate: 10000 # sampling rate: on average 1/rate events are reported; "0" disables profiler prometheus: enabled: true @@ -60,6 +62,9 @@ tree: replication_channel_capacity: 32 replication_timeout: 5s sync_interval: 1h + authorized_keys: # list of hex-encoded public keys that have rights to use the Tree Service with frostfs-cli + - 0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0 + - 02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56 control: authorized_keys: # list of hex-encoded public keys that have rights to use the Control Service @@ -212,3 +217,6 @@ tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost" + +runtime: + soft_memory_limit: 1gb diff --git a/docs/evacuation.md b/docs/evacuation.md new file mode 100644 index 000000000..9bfa0e214 --- /dev/null +++ b/docs/evacuation.md @@ -0,0 +1,92 @@ +# Shard data evacuation + +## Overview + +Evacuation is the process of transferring data from one shard to another. Evacuation is used in case of problems with the shard in order to save data. + +To start the evacuation, it is necessary that the shard is in read-only mode (read more [here](./shard-modes.md)). + +First of all, by the evacuation the data is transferred to other shards of the same node; if it is not possible, then the data is transferred to other nodes. + +Only one running evacuation process is allowed on the node at a time. + +`frostfs-cli` utility is used to manage evacuation. + +## Commands + +`frostfs-cli control shards evacuation start` starts evacuation process for shards specified. To start evacuating all node shards, use the `--all` flag. + +`frostfs-cli control shards evacuation stop` stops running evacuation process. + +`frostfs-cli control shards evacuation status` prints evacuation process status. + +See commands `--help` output for detailed description. + +## Examples + +### Set shard mode to read only +```bash +frostfs-cli control shards set-mode --mode read-only --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 8kEBwtvKLU3Hva3PaaodUi +Enter password > +Shard mode update request successfully sent. +``` + +### Start evacuation and get status +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 8kEBwtvKLU3Hva3PaaodUi +Enter password > +Shard evacuation has been successfully started. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 14 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:03. Estimated time left: 2 minutes. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 23 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:01:05. Estimated time left: 1 minutes. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: completed. Evacuated 61 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:02:13. +``` + +### Stop running evacuation process +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 +Enter password > +Shard evacuation has been successfully started. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 15 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:03. Estimated time left: 0 minutes. + +frostfs-cli control shards evacuation stop --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Evacuation stopped. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: completed. Evacuated 31 object out of 73, failed to evacuate 0 objects. Error: context canceled. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:07. +``` + +### Start evacuation and await it completes +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 --await +Enter password > +Shard evacuation has been successfully started. +Progress will be reported every 5 seconds. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 18 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:04. Estimated time left: 0 minutes. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 43 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:09. Estimated time left: 0 minutes. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 68 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. Estimated time left: 0 minutes. +Shard evacuation has been completed. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. +``` + +### Start evacuation and await it completes without progress notifications +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 --await --no-progress +Enter password > +Shard evacuation has been successfully started. +Shard evacuation has been completed. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:20:00Z UTC. Duration: 00:00:14. +``` diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 366c263a0..439edf598 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -24,6 +24,7 @@ There are some custom types used for brevity: | `policer` | [Policer service configuration](#policer-section) | | `replicator` | [Replicator service configuration](#replicator-section) | | `storage` | [Storage engine configuration](#storage-section) | +| `runtime` | [Runtime configuration](#runtime-section) | # `control` section @@ -75,13 +76,23 @@ element. Contains configuration for the `pprof` profiler. -| Parameter | Type | Default value | Description | -|--------------------|------------|---------------|-----------------------------------------| -| `enabled` | `bool` | `false` | Flag to enable the service. | -| `address` | `string` | | Address that service listener binds to. | -| `shutdown_timeout` | `duration` | `30s` | Time to wait for a graceful shutdown. | +| Parameter | Type | Default value | Description | +|--------------------|-----------------------------------|---------------|-----------------------------------------| +| `enabled` | `bool` | `false` | Flag to enable the service. | +| `address` | `string` | | Address that service listener binds to. | +| `shutdown_timeout` | `duration` | `30s` | Time to wait for a graceful shutdown. | +| `debug` | [Debug config](#debug-subsection) | | Optional profiles configuration | +## `debug` subsection + +Contains optional profiles configuration. + +| Parameter | Type | Default value | Description | +|--------------|-------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `block_rate` | `int` | `0` | Controls the block profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetBlockProfileRate. | +| `mutex_rate` | `int` | `0` | Controls the mutex profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetMutexProfileFraction. | + # `prometheus` section Contains configuration for the `prometheus` metrics service. @@ -157,6 +168,7 @@ Local storage engine configuration. |----------------------------|-----------------------------------|---------------|------------------------------------------------------------------------------------------------------------------| | `shard_pool_size` | `int` | `20` | Pool size for shard workers. Limits the amount of concurrent `PUT` operations on each shard. | | `shard_ro_error_threshold` | `int` | `0` | Maximum amount of storage errors to encounter before shard automatically moves to `Degraded` or `ReadOnly` mode. | +| `low_mem` | `bool` | `false` | Reduce memory consumption by reducing performance. | | `shard` | [Shard config](#shard-subsection) | | Configuration for separate shards. | ## `shard` subsection @@ -415,3 +427,15 @@ object: | `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | | `put.pool_size_remote` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | | `put.pool_size_local` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | + +# `runtime` section +Contains runtime parameters. + +```yaml +runtime: + soft_memory_limit: 1GB +``` + +| Parameter | Type | Default value | Description | +|---------------------|--------|---------------|--------------------------------------------------------------------------| +| `soft_memory_limit` | `size` | 0 | Soft memory limit for the runtime. Zero or no value stands for no limit. If `GOMEMLIMIT` environment variable is set, the value from the configuration file will be ignored. | diff --git a/go.mod b/go.mod index ae0bc88c1..2e7d8fd56 100644 --- a/go.mod +++ b/go.mod @@ -1,116 +1,124 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.18 +go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230503082209-d4fe9a193d1a - git.frostfs.info/TrueCloudLab/hrw v1.2.0 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230719100335-582d94c81c74 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230719130356-5defed4ab435 + git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.3.0 - github.com/hashicorp/golang-lru/v2 v2.0.1 - github.com/klauspost/compress v1.16.5 + github.com/hashicorp/golang-lru/v2 v2.0.4 + github.com/klauspost/compress v1.16.6 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 - github.com/multiformats/go-multiaddr v0.8.0 - github.com/nats-io/nats.go v1.22.1 - github.com/nspcc-dev/neo-go v0.100.1 + github.com/multiformats/go-multiaddr v0.9.0 + github.com/nats-io/nats.go v1.27.1 + github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/olekukonko/tablewriter v0.0.5 - github.com/panjf2000/ants/v2 v2.4.0 - github.com/paulmach/orb v0.2.2 - github.com/prometheus/client_golang v1.15.0 - github.com/prometheus/client_model v0.3.0 - github.com/spf13/cast v1.5.0 - github.com/spf13/cobra v1.6.1 + github.com/panjf2000/ants/v2 v2.7.5 + github.com/paulmach/orb v0.9.2 + github.com/prometheus/client_golang v1.16.0 + github.com/spf13/cast v1.5.1 + github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.15.0 - github.com/stretchr/testify v1.8.2 - go.etcd.io/bbolt v1.3.6 - go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/trace v1.14.0 - go.uber.org/atomic v1.10.0 + github.com/spf13/viper v1.16.0 + github.com/stretchr/testify v1.8.4 + go.etcd.io/bbolt v1.3.7 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 - golang.org/x/sync v0.1.0 - golang.org/x/term v0.5.0 - google.golang.org/grpc v1.53.0 - google.golang.org/protobuf v1.30.0 + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df + golang.org/x/sync v0.3.0 + golang.org/x/term v0.9.0 + google.golang.org/grpc v1.56.1 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 ) require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect - github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect - github.com/benbjohnson/clock v1.1.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/bits-and-blooms/bitset v1.8.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.11.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.3 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.2.0 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect - github.com/ipfs/go-cid v0.3.2 // indirect - github.com/klauspost/cpuid/v2 v2.2.2 // indirect + github.com/holiman/uint256 v1.2.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multihash v0.2.1 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/nats-io/nats-server/v2 v2.7.4 // indirect - github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.7 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect - github.com/twmb/murmur3 v1.1.5 // indirect - github.com/urfave/cli v1.22.5 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect - go.opentelemetry.io/otel/sdk v1.14.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.4.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + github.com/twmb/murmur3 v1.1.8 // indirect + github.com/urfave/cli v1.22.14 // indirect + go.mongodb.org/mongo-driver v1.12.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/sdk v1.16.0 // indirect + go.opentelemetry.io/proto/otlp v0.20.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect + google.golang.org/genproto v0.0.0-20230628200519-e449d1ea0e82 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230628200519-e449d1ea0e82 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230628200519-e449d1ea0e82 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - lukechampine.com/blake3 v1.1.7 // indirect -) - -retract ( - v1.22.1 // Contains retraction only. - v1.22.0 // Published accidentally. + lukechampine.com/blake3 v1.2.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 633ac9a18..c8737ebdc 100644 Binary files a/go.sum and b/go.sum differ diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 9b6e03499..a400187cc 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -52,7 +52,6 @@ const ( PolicerRoutineStopped = "routine stopped" // Info in ../node/pkg/services/policer/process.go PolicerFailureAtObjectSelectForReplication = "failure at object select for replication" // Warn in ../node/pkg/services/policer/process.go PolicerPoolSubmission = "pool submission" // Warn in ../node/pkg/services/policer/process.go - PolicerTuneReplicationCapacity = "tune replication capacity" // Debug in ../node/pkg/services/policer/process.go ReplicatorFinishWork = "finish work" // Debug in ../node/pkg/services/replicator/process.go ReplicatorCouldNotGetObjectFromLocalStorage = "could not get object from local storage" // Error in ../node/pkg/services/replicator/process.go ReplicatorCouldNotReplicateObject = "could not replicate object" // Error in ../node/pkg/services/replicator/process.go @@ -195,7 +194,6 @@ const ( EventIgnoreNilNotaryEventHandler = "ignore nil notary event handler" // Warn in ../node/pkg/morph/event/listener.go EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" // Warn in ../node/pkg/morph/event/listener.go EventIgnoreNilBlockHandler = "ignore nil block handler" // Warn in ../node/pkg/morph/event/listener.go - SubscriberUnsubscribeForNotification = "unsubscribe for notification" // Error in ../node/pkg/morph/subscriber/subscriber.go SubscriberRemoteNotificationChannelHasBeenClosed = "remote notification channel has been closed" // Warn in ../node/pkg/morph/subscriber/subscriber.go SubscriberCantCastNotifyEventValueToTheNotifyStruct = "can't cast notify event value to the notify struct" // Error in ../node/pkg/morph/subscriber/subscriber.go SubscriberNewNotificationEventFromSidechain = "new notification event from sidechain" // Debug in ../node/pkg/morph/subscriber/subscriber.go @@ -415,6 +413,7 @@ const ( FrostFSIRApplicationStopped = "application stopped" // Info in ../node/cmd/frostfs-ir/main.go FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" // Debug in ../node/pkg/morph/client/constructor.go FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" // Info in ../node/pkg/morph/client/constructor.go + FrostFSIRReloadExtraWallets = "reload extra wallets" // Info in ../node/cmd/frostfs-ir/config.go FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" // Error in ../node/cmd/frostfs-node/grpc.go FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" // Error in ../node/cmd/frostfs-node/grpc.go FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go @@ -477,5 +476,23 @@ const ( FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" // Debug in ../node/cmd/frostfs-node/tree.go FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" // Error in ../node/cmd/frostfs-node/tree.go FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" // Error in ../node/cmd/frostfs-node/control.go - CommonApplicationStarted = "application started" // Info in ../node/cmd/frostfs-ir/main.go + FrostFSNodePolicerIsDisabled = "policer is disabled" + CommonApplicationStarted = "application started" + ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" + ShardGCCollectingExpiredObjectsCompleted = "collecting expired objects completed" + ShardGCCollectingExpiredLocksStarted = "collecting expired locks started" + ShardGCCollectingExpiredLocksCompleted = "collecting expired locks completed" + ShardGCRemoveGarbageStarted = "garbage remove started" + ShardGCRemoveGarbageCompleted = "garbage remove completed" + EngineShardsEvacuationFailedToCount = "failed to get total objects count to evacuate" + EngineShardsEvacuationFailedToListObjects = "failed to list objects to evacuate" + EngineShardsEvacuationFailedToReadObject = "failed to read object to evacuate" + EngineShardsEvacuationFailedToMoveObject = "failed to evacuate object to other node" + ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" + ShardDeleteCantDeleteFromWriteCache = "can't delete object from write cache" + FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" + EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" + NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap = "the node is already in candidate list with online state, skip initial bootstrap" + RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" + RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" ) diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index bd186006e..8c92901f2 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -12,7 +12,8 @@ import ( // node's client. type Client interface { ContainerAnnounceUsedSpace(context.Context, client.PrmAnnounceSpace) (*client.ResAnnounceSpace, error) - ObjectPutInit(context.Context, client.PrmObjectPutInit) (*client.ObjectWriter, error) + ObjectPutInit(context.Context, client.PrmObjectPutInit) (client.ObjectWriter, error) + ObjectPutSingle(context.Context, client.PrmObjectPutSingle) (*client.ResObjectPutSingle, error) ObjectDelete(context.Context, client.PrmObjectDelete) (*client.ResObjectDelete, error) ObjectGetInit(context.Context, client.PrmObjectGet) (*client.ObjectReader, error) ObjectHead(context.Context, client.PrmObjectHead) (*client.ResObjectHead, error) diff --git a/pkg/core/container/delete.go b/pkg/core/container/delete.go index e3379446f..8e0aaebb9 100644 --- a/pkg/core/container/delete.go +++ b/pkg/core/container/delete.go @@ -1,6 +1,7 @@ package container import ( + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" ) @@ -8,43 +9,14 @@ import ( // RemovalWitness groups the information required // to prove and verify the removal of a container. type RemovalWitness struct { - cnr cid.ID + // ContainerID returns the identifier of the container + // to be removed. + ContainerID cid.ID - sig []byte + // Signature the signature of the container identifier. + Signature *refs.Signature - token *session.Container -} - -// ContainerID returns the identifier of the container -// to be removed. -func (x RemovalWitness) ContainerID() cid.ID { - return x.cnr -} - -// SetContainerID sets the identifier of the container -// to be removed. -func (x *RemovalWitness) SetContainerID(id cid.ID) { - x.cnr = id -} - -// Signature returns the signature of the container identifier. -func (x RemovalWitness) Signature() []byte { - return x.sig -} - -// SetSignature sets a signature of the container identifier. -func (x *RemovalWitness) SetSignature(sig []byte) { - x.sig = sig -} - -// SessionToken returns the token of the session within -// which the container was removed. -func (x RemovalWitness) SessionToken() *session.Container { - return x.token -} - -// SetSessionToken sets the token of the session within -// which the container was removed. -func (x *RemovalWitness) SetSessionToken(tok *session.Container) { - x.token = tok + // SessionToken the token of the session within + // which the container was removed. + SessionToken *session.Container } diff --git a/pkg/core/object/address.go b/pkg/core/object/address.go index cd5559d9f..12e5c89ce 100644 --- a/pkg/core/object/address.go +++ b/pkg/core/object/address.go @@ -1,7 +1,7 @@ package object import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -9,5 +9,5 @@ import ( // object type. type AddressWithType struct { Address oid.Address - Type object.Type + Type objectSDK.Type } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 946cfc462..e6d8174fa 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -12,7 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -64,6 +64,8 @@ var errNoExpirationEpoch = errors.New("missing expiration epoch attribute") var errTombstoneExpiration = errors.New("tombstone body and header contain different expiration values") +var errMissingSignature = errors.New("missing signature") + func defaultCfg() *cfg { return new(cfg) } @@ -87,7 +89,7 @@ func NewFormatValidator(opts ...FormatValidatorOption) *FormatValidator { // If unprepared is true, only fields set by user are validated. // // Returns nil error if the object has valid structure. -func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unprepared bool) error { +func (v *FormatValidator) Validate(ctx context.Context, obj *objectSDK.Object, unprepared bool) error { if obj == nil { return errNilObject } @@ -119,7 +121,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unpr return fmt.Errorf("object did not pass expiration check: %w", err) } - if err := object.CheckHeaderVerificationFields(obj); err != nil { + if err := objectSDK.CheckHeaderVerificationFields(obj); err != nil { return fmt.Errorf("(%T) could not validate header fields: %w", v, err) } } @@ -132,12 +134,10 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unpr return nil } -func (v *FormatValidator) validateSignatureKey(obj *object.Object) error { - // FIXME(@cthulhu-rider): temp solution, see neofs-sdk-go#233 +func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { sig := obj.Signature() if sig == nil { - // TODO(@cthulhu-rider): #1387 use "const" error - return errors.New("missing signature") + return errMissingSignature } var sigV2 refs.Signature @@ -158,8 +158,6 @@ func (v *FormatValidator) validateSignatureKey(obj *object.Object) error { return v.checkOwnerKey(*obj.OwnerID(), key) } - // FIXME: #1159 perform token verification - return nil } @@ -179,13 +177,13 @@ func (v *FormatValidator) checkOwnerKey(id user.ID, key frostfsecdsa.PublicKey) // - object.TypeTombstone; // - object.TypeLock. type ContentMeta struct { - typ object.Type + typ objectSDK.Type objs []oid.ID } // Type returns object's type. -func (i ContentMeta) Type() object.Type { +func (i ContentMeta) Type() objectSDK.Type { return i.typ } @@ -198,17 +196,17 @@ func (i ContentMeta) Objects() []oid.ID { } // ValidateContent validates payload content according to the object type. -func (v *FormatValidator) ValidateContent(o *object.Object) (ContentMeta, error) { +func (v *FormatValidator) ValidateContent(o *objectSDK.Object) (ContentMeta, error) { meta := ContentMeta{ typ: o.Type(), } switch o.Type() { - case object.TypeTombstone: + case objectSDK.TypeTombstone: if err := v.fillAndValidateTombstoneMeta(o, &meta); err != nil { return ContentMeta{}, err } - case object.TypeLock: + case objectSDK.TypeLock: if err := v.fillAndValidateLockMeta(o, &meta); err != nil { return ContentMeta{}, err } @@ -219,7 +217,7 @@ func (v *FormatValidator) ValidateContent(o *object.Object) (ContentMeta, error) return meta, nil } -func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *ContentMeta) error { +func (v *FormatValidator) fillAndValidateLockMeta(o *objectSDK.Object, meta *ContentMeta) error { if len(o.Payload()) == 0 { return errors.New("empty payload in lock") } @@ -241,7 +239,7 @@ func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *Conten return fmt.Errorf("lock object expiration: %d; current: %d", lockExp, currEpoch) } - var lock object.Lock + var lock objectSDK.Lock if err = lock.Unmarshal(o.Payload()); err != nil { return fmt.Errorf("decode lock payload: %w", err) @@ -257,12 +255,12 @@ func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *Conten return nil } -func (v *FormatValidator) fillAndValidateTombstoneMeta(o *object.Object, meta *ContentMeta) error { +func (v *FormatValidator) fillAndValidateTombstoneMeta(o *objectSDK.Object, meta *ContentMeta) error { if len(o.Payload()) == 0 { return fmt.Errorf("(%T) empty payload in tombstone", v) } - tombstone := object.NewTombstone() + tombstone := objectSDK.NewTombstone() if err := tombstone.Unmarshal(o.Payload()); err != nil { return fmt.Errorf("(%T) could not unmarshal tombstone content: %w", v, err) @@ -288,7 +286,7 @@ func (v *FormatValidator) fillAndValidateTombstoneMeta(o *object.Object, meta *C var errExpired = errors.New("object has expired") -func (v *FormatValidator) checkExpiration(ctx context.Context, obj *object.Object) error { +func (v *FormatValidator) checkExpiration(ctx context.Context, obj *objectSDK.Object) error { exp, err := expirationEpochAttribute(obj) if err != nil { if errors.Is(err, errNoExpirationEpoch) { @@ -322,7 +320,7 @@ func (v *FormatValidator) checkExpiration(ctx context.Context, obj *object.Objec return nil } -func expirationEpochAttribute(obj *object.Object) (uint64, error) { +func expirationEpochAttribute(obj *objectSDK.Object) (uint64, error) { for _, a := range obj.Attributes() { if a.Key() != objectV2.SysAttributeExpEpoch && a.Key() != objectV2.SysAttributeExpEpochNeoFS { continue @@ -339,7 +337,7 @@ var ( errEmptyAttrVal = errors.New("empty attribute value") ) -func (v *FormatValidator) checkAttributes(obj *object.Object) error { +func (v *FormatValidator) checkAttributes(obj *objectSDK.Object) error { as := obj.Attributes() mUnique := make(map[string]struct{}, len(as)) @@ -363,7 +361,7 @@ func (v *FormatValidator) checkAttributes(obj *object.Object) error { var errIncorrectOwner = errors.New("incorrect object owner") -func (v *FormatValidator) checkOwner(obj *object.Object) error { +func (v *FormatValidator) checkOwner(obj *objectSDK.Object) error { if idOwner := obj.OwnerID(); idOwner == nil || len(idOwner.WalletBytes()) == 0 { return errIncorrectOwner } diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index 2cf5099ba..d04c16709 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -8,7 +8,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" sessiontest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session/test" @@ -17,11 +17,11 @@ import ( "github.com/stretchr/testify/require" ) -func blankValidObject(key *ecdsa.PrivateKey) *object.Object { +func blankValidObject(key *ecdsa.PrivateKey) *objectSDK.Object { var idOwner user.ID user.IDFromKey(&idOwner, key.PublicKey) - obj := object.New() + obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetOwnerID(&idOwner) @@ -66,20 +66,20 @@ func TestFormatValidator_Validate(t *testing.T) { }) t.Run("nil identifier", func(t *testing.T) { - obj := object.New() + obj := objectSDK.New() require.ErrorIs(t, v.Validate(context.Background(), obj, false), errNilID) }) t.Run("nil container identifier", func(t *testing.T) { - obj := object.New() + obj := objectSDK.New() obj.SetID(oidtest.ID()) require.ErrorIs(t, v.Validate(context.Background(), obj, true), errNilCID) }) t.Run("unsigned object", func(t *testing.T) { - obj := object.New() + obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetID(oidtest.ID()) @@ -94,12 +94,12 @@ func TestFormatValidator_Validate(t *testing.T) { err := tok.Sign(ownerKey.PrivateKey) require.NoError(t, err) - obj := object.New() + obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetSessionToken(tok) obj.SetOwnerID(&idOwner) - require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) + require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) require.NoError(t, v.Validate(context.Background(), obj, false)) }) @@ -107,20 +107,20 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("correct w/o session token", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) - require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) + require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) require.NoError(t, v.Validate(context.Background(), obj, false)) }) t.Run("tombstone content", func(t *testing.T) { - obj := object.New() - obj.SetType(object.TypeTombstone) + obj := objectSDK.New() + obj.SetType(objectSDK.TypeTombstone) obj.SetContainerID(cidtest.ID()) _, err := v.ValidateContent(obj) require.Error(t, err) // no tombstone content - content := object.NewTombstone() + content := objectSDK.NewTombstone() content.SetMembers([]oid.ID{oidtest.ID()}) data, err := content.Marshal() @@ -141,7 +141,7 @@ func TestFormatValidator_Validate(t *testing.T) { _, err = v.ValidateContent(obj) require.Error(t, err) // no expiration epoch in tombstone - var expirationAttribute object.Attribute + var expirationAttribute objectSDK.Attribute expirationAttribute.SetKey(objectV2.SysAttributeExpEpoch) expirationAttribute.SetValue(strconv.Itoa(10)) @@ -163,20 +163,20 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, err) // all good require.EqualValues(t, []oid.ID{id}, contentGot.Objects()) - require.Equal(t, object.TypeTombstone, contentGot.Type()) + require.Equal(t, objectSDK.TypeTombstone, contentGot.Type()) }) t.Run("expiration", func(t *testing.T) { - fn := func(val string) *object.Object { + fn := func(val string) *objectSDK.Object { obj := blankValidObject(&ownerKey.PrivateKey) - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(val) obj.SetAttributes(a) - require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) + require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) return obj } @@ -221,11 +221,11 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("duplication", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) - var a1 object.Attribute + var a1 objectSDK.Attribute a1.SetKey("key1") a1.SetValue("val1") - var a2 object.Attribute + var a2 objectSDK.Attribute a2.SetKey("key2") a2.SetValue("val2") @@ -244,7 +244,7 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("empty value", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) - var a object.Attribute + var a objectSDK.Attribute a.SetKey("key") obj.SetAttributes(a) diff --git a/pkg/core/object/object.go b/pkg/core/object/object.go index 8fb656acd..9c450966c 100644 --- a/pkg/core/object/object.go +++ b/pkg/core/object/object.go @@ -1,12 +1,12 @@ package object import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) // AddressOf returns the address of the object. -func AddressOf(obj *object.Object) oid.Address { +func AddressOf(obj *objectSDK.Object) oid.Address { var addr oid.Address id, ok := obj.ID() diff --git a/pkg/innerring/config/fee.go b/pkg/innerring/config/fee.go index d77685643..a26a7bcc6 100644 --- a/pkg/innerring/config/fee.go +++ b/pkg/innerring/config/fee.go @@ -8,19 +8,15 @@ import ( // FeeConfig is an instance that returns extra fee values for contract // invocations without notary support. type FeeConfig struct { - registerNamedCnr, mainchain, sidechain fixedn.Fixed8 } // NewFeeConfig constructs FeeConfig from viper.Viper instance. Latter must not be nil. -// -// Fee for named container registration is taken from "fee.named_container_register" value. func NewFeeConfig(v *viper.Viper) *FeeConfig { return &FeeConfig{ - registerNamedCnr: fixedn.Fixed8(v.GetInt64("fee.named_container_register")), - mainchain: fixedn.Fixed8(v.GetInt64("fee.main_chain")), - sidechain: fixedn.Fixed8(v.GetInt64("fee.side_chain")), + mainchain: fixedn.Fixed8(v.GetInt64("fee.main_chain")), + sidechain: fixedn.Fixed8(v.GetInt64("fee.side_chain")), } } @@ -31,8 +27,3 @@ func (f FeeConfig) MainChainFee() fixedn.Fixed8 { func (f FeeConfig) SideChainFee() fixedn.Fixed8 { return f.sidechain } - -// NamedContainerRegistrationFee returns additional GAS fee for named container registration in FrostFS network. -func (f FeeConfig) NamedContainerRegistrationFee() fixedn.Fixed8 { - return f.registerNamedCnr -} diff --git a/pkg/innerring/config/fee_test.go b/pkg/innerring/config/fee_test.go index a0c56aac1..f7330c6ca 100644 --- a/pkg/innerring/config/fee_test.go +++ b/pkg/innerring/config/fee_test.go @@ -18,7 +18,6 @@ func TestConfig(t *testing.T) { fee: main_chain: 50000000 side_chain: 200000000 - named_container_register: 2500000000 `, ) v := viper.New() @@ -29,7 +28,6 @@ fee: config := NewFeeConfig(v) require.Equal(t, fixedn.Fixed8(50000000), config.MainChainFee(), "main chain fee invalid") require.Equal(t, fixedn.Fixed8(200000000), config.SideChainFee(), "side chain fee invalid") - require.Equal(t, fixedn.Fixed8(2500000000), config.NamedContainerRegistrationFee(), "named container register fee invalid") }) t.Run("nothing set", func(t *testing.T) { @@ -43,7 +41,6 @@ fee: config := NewFeeConfig(v) require.Equal(t, fixedn.Fixed8(0), config.MainChainFee(), "main chain fee invalid") require.Equal(t, fixedn.Fixed8(0), config.SideChainFee(), "side chain fee invalid") - require.Equal(t, fixedn.Fixed8(0), config.NamedContainerRegistrationFee(), "named container register fee invalid") }) t.Run("partially set", func(t *testing.T) { @@ -62,7 +59,6 @@ fee: config := NewFeeConfig(v) require.Equal(t, fixedn.Fixed8(10), config.MainChainFee(), "main chain fee invalid") require.Equal(t, fixedn.Fixed8(0), config.SideChainFee(), "side chain fee invalid") - require.Equal(t, fixedn.Fixed8(0), config.NamedContainerRegistrationFee(), "named container register fee invalid") }) } diff --git a/pkg/innerring/contracts.go b/pkg/innerring/contracts.go index 55c2ff582..4a80296f4 100644 --- a/pkg/innerring/contracts.go +++ b/pkg/innerring/contracts.go @@ -22,7 +22,7 @@ type contracts struct { alphabet AlphabetContracts // in morph } -func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, withoutMainNotary, withoutSideNotary bool) (*contracts, error) { +func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, withoutMainNotary bool) (*contracts, error) { var ( result = new(contracts) err error @@ -42,11 +42,9 @@ func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, without } } - if !withoutSideNotary { - result.proxy, err = parseContract(cfg, morph, "contracts.proxy", client.NNSProxyContractName) - if err != nil { - return nil, fmt.Errorf("can't get proxy script hash: %w", err) - } + result.proxy, err = parseContract(cfg, morph, "contracts.proxy", client.NNSProxyContractName) + if err != nil { + return nil, fmt.Errorf("can't get proxy script hash: %w", err) } targets := [...]struct { diff --git a/pkg/innerring/contracts_test.go b/pkg/innerring/contracts_test.go index e04e22f06..5f877133c 100644 --- a/pkg/innerring/contracts_test.go +++ b/pkg/innerring/contracts_test.go @@ -35,7 +35,7 @@ contracts: t.Run("all enabled", func(t *testing.T) { t.Parallel() - c, err := parseContracts(v, nil, false, false, false) + c, err := parseContracts(v, nil, false, false) require.NoError(t, err, "failed to parse contracts") frostfsExp, _ := util.Uint160DecodeStringLE("ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62") @@ -70,7 +70,7 @@ contracts: t.Run("all disabled", func(t *testing.T) { t.Parallel() - c, err := parseContracts(v, nil, true, true, true) + c, err := parseContracts(v, nil, true, true) require.NoError(t, err, "failed to parse contracts") require.Equal(t, util.Uint160{}, c.frostfs, "invalid frostfs") @@ -89,7 +89,8 @@ contracts: netmapIDExp, _ := util.Uint160DecodeStringLE("83c600c81d47a1b1b7cf58eb49ae7ee7240dc742") require.Equal(t, netmapIDExp, c.netmap, "invalid netmap") - require.Equal(t, util.Uint160{}, c.proxy, "invalid proxy") + proxyExp, _ := util.Uint160DecodeStringLE("abc8794bb40a21f2db5f21ae62741eb46c8cad1c") + require.Equal(t, proxyExp, c.proxy, "invalid proxy") require.Equal(t, 2, len(c.alphabet), "invalid alphabet contracts length") @@ -100,9 +101,9 @@ contracts: require.Equal(t, bukyExp, c.alphabet[buky], "invalid buky") }) - t.Run("main notary & side notary disabled", func(t *testing.T) { + t.Run("main notary disabled", func(t *testing.T) { t.Parallel() - c, err := parseContracts(v, nil, false, true, true) + c, err := parseContracts(v, nil, false, true) require.NoError(t, err, "failed to parse contracts") frostfsExp, _ := util.Uint160DecodeStringLE("ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62") @@ -122,7 +123,8 @@ contracts: netmapIDExp, _ := util.Uint160DecodeStringLE("83c600c81d47a1b1b7cf58eb49ae7ee7240dc742") require.Equal(t, netmapIDExp, c.netmap, "invalid netmap") - require.Equal(t, util.Uint160{}, c.proxy, "invalid proxy") + proxyExp, _ := util.Uint160DecodeStringLE("abc8794bb40a21f2db5f21ae62741eb46c8cad1c") + require.Equal(t, proxyExp, c.proxy, "invalid proxy") require.Equal(t, 2, len(c.alphabet), "invalid alphabet contracts length") @@ -159,7 +161,7 @@ contracts: err := v.ReadConfig(file) require.NoError(t, err, "read config file failed") - _, err = parseContracts(v, nil, false, false, false) + _, err = parseContracts(v, nil, false, false) require.Error(t, err, "unexpected success") }) @@ -196,7 +198,7 @@ contracts: }, } - _, err = parseContracts(v, morph, false, false, false) + _, err = parseContracts(v, morph, false, false) require.ErrorContains(t, err, "could not read all contracts: required 3, read 2", "unexpected success") }) } diff --git a/pkg/innerring/indexer_test.go b/pkg/innerring/indexer_test.go index 493ae92de..1937f7a49 100644 --- a/pkg/innerring/indexer_test.go +++ b/pkg/innerring/indexer_test.go @@ -2,12 +2,12 @@ package innerring import ( "fmt" + "sync/atomic" "testing" "time" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func TestIndexerReturnsIndexes(t *testing.T) { @@ -209,7 +209,7 @@ type testCommiteeFetcher struct { } func (f *testCommiteeFetcher) Committee() (keys.PublicKeys, error) { - f.calls.Inc() + f.calls.Add(1) return f.keys, f.err } @@ -220,6 +220,6 @@ type testIRFetcher struct { } func (f *testIRFetcher) InnerRingKeys() (keys.PublicKeys, error) { - f.calls.Inc() + f.calls.Add(1) return f.keys, f.err } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 84d08c4c6..52ffb10a1 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -16,7 +16,6 @@ import ( nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation" addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress" statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" @@ -53,6 +52,7 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, s.netmapProcessor, err = netmap.New(&netmap.Params{ Log: s.log, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.netmap"), NetmapClient: netmap.NewNetmapClient(s.netmapClient), EpochTimer: s, @@ -72,7 +72,6 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, addrvalidator.New(), locodeValidator, ), - NotaryDisabled: s.sideNotaryConfig.disabled, NodeStateSettings: netSettings, }) @@ -118,18 +117,16 @@ func (s *Server) initMainnet(ctx context.Context, cfg *viper.Viper, morphChain * } func (s *Server) enableNotarySupport() error { - if !s.sideNotaryConfig.disabled { - // enable notary support in the side client - err := s.morphClient.EnableNotarySupport( - client.WithProxyContract(s.contracts.proxy), - ) - if err != nil { - return fmt.Errorf("could not enable side chain notary support: %w", err) - } - - s.morphListener.EnableNotarySupport(s.contracts.proxy, s.morphClient.Committee, s.morphClient) + // enable notary support in the side client + err := s.morphClient.EnableNotarySupport( + client.WithProxyContract(s.contracts.proxy), + ) + if err != nil { + return fmt.Errorf("could not enable side chain notary support: %w", err) } + s.morphListener.EnableNotarySupport(s.contracts.proxy, s.morphClient.Committee, s.morphClient) + if !s.mainNotaryConfig.disabled { // enable notary support in the main client err := s.mainnetClient.EnableNotarySupport( @@ -145,13 +142,12 @@ func (s *Server) enableNotarySupport() error { } func (s *Server) initNotaryConfig() { - s.mainNotaryConfig, s.sideNotaryConfig = notaryConfigs( - s.morphClient.ProbeNotary(), + s.mainNotaryConfig = notaryConfigs( !s.withoutMainNet && s.mainnetClient.ProbeNotary(), // if mainnet disabled then notary flag must be disabled too ) s.log.Info(logs.InnerringNotarySupport, - zap.Bool("sidechain_enabled", !s.sideNotaryConfig.disabled), + zap.Bool("sidechain_enabled", true), zap.Bool("mainchain_enabled", !s.mainNotaryConfig.disabled), ) } @@ -166,16 +162,16 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli } else { // create governance processor governanceProcessor, err := governance.New(&governance.Params{ - Log: s.log, - FrostFSClient: frostfsCli, - NetmapClient: s.netmapClient, - AlphabetState: s, - EpochState: s, - Voter: s, - IRFetcher: irf, - MorphClient: s.morphClient, - MainnetClient: s.mainnetClient, - NotaryDisabled: s.sideNotaryConfig.disabled, + Log: s.log, + Metrics: s.irMetrics, + FrostFSClient: frostfsCli, + NetmapClient: s.netmapClient, + AlphabetState: s, + EpochState: s, + Voter: s, + IRFetcher: irf, + MorphClient: s.morphClient, + MainnetClient: s.mainnetClient, }) if err != nil { return nil, err @@ -205,7 +201,7 @@ func (s *Server) createIRFetcher() irFetcher { return irf } -func (s *Server) initTimers(cfg *viper.Viper, processors *serverProcessors, morphClients *serverMorphClients) { +func (s *Server) initTimers(cfg *viper.Viper, morphClients *serverMorphClients) { s.epochTimer = newEpochTimer(&epochTimerArgs{ l: s.log, alphabetState: s, @@ -220,23 +216,24 @@ func (s *Server) initTimers(cfg *viper.Viper, processors *serverProcessors, morp // initialize emission timer emissionTimer := newEmissionTimer(&emitTimerArgs{ - ap: processors.AlphabetProcessor, + ap: s.alphabetProcessor, emitDuration: cfg.GetUint32("timers.emit"), }) s.addBlockTimer(emissionTimer) } -func (s *Server) initAlphabetProcessor(cfg *viper.Viper) (*alphabet.Processor, error) { +func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { parsedWallets, err := parseWalletAddressesFromStrings(cfg.GetStringSlice("emit.extra_wallets")) if err != nil { - return nil, err + return err } // create alphabet processor - alphabetProcessor, err := alphabet.New(&alphabet.Params{ + s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, Log: s.log, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.alphabet"), AlphabetContracts: s.contracts.alphabet, NetmapClient: s.netmapClient, @@ -245,15 +242,15 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) (*alphabet.Processor, e StorageEmission: cfg.GetUint64("emit.storage.amount"), }) if err != nil { - return nil, err + return err } - err = bindMorphProcessor(alphabetProcessor, s) + err = bindMorphProcessor(s.alphabetProcessor, s) if err != nil { - return nil, err + return err } - return alphabetProcessor, nil + return nil } func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client, @@ -261,12 +258,13 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.container"), AlphabetState: s, ContainerClient: cnrClient, + MorphClient: cnrClient.Morph(), FrostFSIDClient: frostfsIDClient, NetworkState: s.netmapClient, - NotaryDisabled: s.sideNotaryConfig.disabled, }) if err != nil { return err @@ -279,6 +277,7 @@ func (s *Server) initBalanceProcessor(cfg *viper.Viper, frostfsCli *frostfsClien // create balance processor balanceProcessor, err := balance.New(&balance.Params{ Log: s.log, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.balance"), FrostFSClient: frostfsCli, BalanceSC: s.contracts.balance, @@ -299,6 +298,7 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper, frostfsIDClient * frostfsProcessor, err := frostfs.New(&frostfs.Params{ Log: s.log, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.frostfs"), FrostFSContract: s.contracts.frostfs, FrostFSIDClient: frostfsIDClient, @@ -389,14 +389,6 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { container.AsAlphabet(), ) - if s.sideNotaryConfig.disabled { - // in non-notary environments we customize fee for named container registration - // because it takes much more additional GAS than other operations. - morphCnrOpts = append(morphCnrOpts, - container.WithCustomFeeForNamedPut(s.feeConfig.NamedContainerRegistrationFee()), - ) - } - result.CnrClient, err = container.NewFromMorph(s.morphClient, s.contracts.container, fee, morphCnrOpts...) if err != nil { return nil, err @@ -426,13 +418,7 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { return result, nil } -type serverProcessors struct { - AlphabetProcessor *alphabet.Processor -} - -func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClients) (*serverProcessors, error) { - result := &serverProcessors{} - +func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClients) error { irf := s.createIRFetcher() s.statusIndex = newInnerRingIndexer( @@ -444,35 +430,35 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien alphaSync, err := s.createAlphaSync(cfg, morphClients.FrostFSClient, irf) if err != nil { - return nil, err + return err } err = s.initNetmapProcessor(cfg, morphClients.CnrClient, alphaSync) if err != nil { - return nil, err + return err } err = s.initContainerProcessor(cfg, morphClients.CnrClient, morphClients.FrostFSIDClient) if err != nil { - return nil, err + return err } err = s.initBalanceProcessor(cfg, morphClients.FrostFSClient) if err != nil { - return nil, err + return err } err = s.initFrostFSMainnetProcessor(cfg, morphClients.FrostFSIDClient) if err != nil { - return nil, err + return err } - result.AlphabetProcessor, err = s.initAlphabetProcessor(cfg) + err = s.initAlphabetProcessor(cfg) if err != nil { - return nil, err + return err } - return result, nil + return nil } func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- error) (*chainParams, error) { @@ -483,11 +469,12 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- } morphChain := &chainParams{ - log: s.log, - cfg: cfg, - key: s.key, - name: morphPrefix, - from: fromSideChainBlock, + log: s.log, + cfg: cfg, + key: s.key, + name: morphPrefix, + from: fromSideChainBlock, + morphCacheMetric: s.irMetrics.MorphCacheMetrics(), } // create morph client @@ -516,7 +503,6 @@ func (s *Server) initContracts(cfg *viper.Viper) error { s.morphClient, s.withoutMainNet, s.mainNotaryConfig.disabled, - s.sideNotaryConfig.disabled, ) return err @@ -535,11 +521,3 @@ func (s *Server) initKey(cfg *viper.Viper) error { s.key = acc.PrivateKey() return nil } - -func (s *Server) initMetrics(cfg *viper.Viper) { - if cfg.GetString("prometheus.address") == "" { - return - } - m := metrics.NewInnerRingMetrics() - s.metrics = &m -} diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index deb546f08..1567e40d3 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -5,9 +5,11 @@ import ( "errors" "fmt" "io" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" @@ -28,7 +30,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/viper" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -51,18 +52,17 @@ type ( epochDuration atomic.Uint64 statusIndex *innerRingIndexer precision precision.Fixed8Converter - healthStatus atomic.Value + healthStatus atomic.Int32 balanceClient *balanceClient.Client netmapClient *nmClient.Client persistate *state.PersistentStorage // metrics - metrics *metrics.InnerRingServiceMetrics + irMetrics *metrics.InnerRingServiceMetrics // notary configuration feeConfig *config.FeeConfig mainNotaryConfig *notaryConfig - sideNotaryConfig *notaryConfig // internal variables key *keys.PrivateKey @@ -73,7 +73,8 @@ type ( withoutMainNet bool // runtime processors - netmapProcessor *netmap.Processor + netmapProcessor *netmap.Processor + alphabetProcessor *alphabet.Processor workers []func(context.Context) @@ -101,12 +102,13 @@ type ( } chainParams struct { - log *logger.Logger - cfg *viper.Viper - key *keys.PrivateKey - name string - sgn *transaction.Signer - from uint32 // block height + log *logger.Logger + cfg *viper.Viper + key *keys.PrivateKey + name string + sgn *transaction.Signer + from uint32 // block height + morphCacheMetric metrics.MorphCacheMetrics } ) @@ -265,14 +267,11 @@ func (s *Server) initMainNotary(ctx context.Context) error { } func (s *Server) initSideNotary(ctx context.Context) error { - if !s.sideNotaryConfig.disabled { - return s.initNotary(ctx, - s.depositSideNotary, - s.awaitSideNotaryDeposit, - "waiting to accept side notary deposit", - ) - } - return nil + return s.initNotary(ctx, + s.depositSideNotary, + s.awaitSideNotaryDeposit, + "waiting to accept side notary deposit", + ) } func (s *Server) tickInitialExpoch() { @@ -328,7 +327,10 @@ func (s *Server) registerStarter(f func() error) { // New creates instance of inner ring sever structure. func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error) (*Server, error) { var err error - server := &Server{log: log} + server := &Server{ + log: log, + irMetrics: metrics.NewInnerRingMetrics(), + } server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED) @@ -383,21 +385,18 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - var processors *serverProcessors - processors, err = server.initProcessors(cfg, morphClients) + err = server.initProcessors(cfg, morphClients) if err != nil { return nil, err } - server.initTimers(cfg, processors, morphClients) + server.initTimers(cfg, morphClients) err = server.initGRPCServer(cfg) if err != nil { return nil, err } - server.initMetrics(cfg) - return server, nil } @@ -467,6 +466,7 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c errChan <- fmt.Errorf("%s chain connection has been lost", p.name) }), client.WithSwitchInterval(p.cfg.GetDuration(p.name+".switch_interval")), + client.WithMorphCacheMetrics(p.morphCacheMetric), ) } @@ -589,3 +589,12 @@ func (s *Server) newEpochTickHandlers() []newEpochHandler { return newEpochHandlers } + +func (s *Server) SetExtraWallets(cfg *viper.Viper) error { + parsedWallets, err := parseWalletAddressesFromStrings(cfg.GetStringSlice("emit.extra_wallets")) + if err != nil { + return err + } + s.alphabetProcessor.SetParsedWallets(parsedWallets) + return nil +} diff --git a/pkg/innerring/metrics/metrics.go b/pkg/innerring/metrics/metrics.go new file mode 100644 index 000000000..002f3afe1 --- /dev/null +++ b/pkg/innerring/metrics/metrics.go @@ -0,0 +1,15 @@ +package metrics + +import "time" + +type Register interface { + SetEpoch(epoch uint64) + SetHealth(s int32) + AddEvent(d time.Duration, typ string, success bool) +} + +type DefaultRegister struct{} + +func (DefaultRegister) SetEpoch(uint64) {} +func (DefaultRegister) SetHealth(int32) {} +func (DefaultRegister) AddEvent(time.Duration, string, bool) {} diff --git a/pkg/innerring/notary.go b/pkg/innerring/notary.go index 30916cb99..c601f5587 100644 --- a/pkg/innerring/notary.go +++ b/pkg/innerring/notary.go @@ -57,11 +57,8 @@ func (s *Server) notaryHandler(_ event.Event) { } } - if !s.sideNotaryConfig.disabled { - _, err := s.depositSideNotary() - if err != nil { - s.log.Error(logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) - } + if _, err := s.depositSideNotary(); err != nil { + s.log.Error(logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) } } @@ -115,16 +112,8 @@ func awaitNotaryDepositInClient(ctx context.Context, cli *client.Client, txHash return errDepositTimeout } -func notaryConfigs(withSideNotary, withMainNotary bool) (main, side *notaryConfig) { +func notaryConfigs(withMainNotary bool) (main *notaryConfig) { main = new(notaryConfig) - side = new(notaryConfig) - - if !withSideNotary { - main.disabled = true - side.disabled = true - - return - } main.disabled = !withMainNotary diff --git a/pkg/innerring/processors/alphabet/handlers.go b/pkg/innerring/processors/alphabet/handlers.go index c0668a4f9..9de075f17 100644 --- a/pkg/innerring/processors/alphabet/handlers.go +++ b/pkg/innerring/processors/alphabet/handlers.go @@ -2,6 +2,7 @@ package alphabet import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "go.uber.org/zap" @@ -13,7 +14,7 @@ func (ap *Processor) HandleGasEmission(ev event.Event) { // send event to the worker pool - err := ap.pool.Submit(func() { ap.processEmit() }) + err := processors.SubmitEvent(ap.pool, ap.metrics, "alphabet_emit_gas", ap.processEmit) if err != nil { // there system can be moved into controlled degradation stage ap.log.Warn(logs.AlphabetAlphabetProcessorWorkerPoolDrained, diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index b8d65dbc5..8a2336011 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -7,17 +7,18 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" ) const emitMethod = "emit" -func (ap *Processor) processEmit() { +func (ap *Processor) processEmit() bool { index := ap.irList.AlphabetIndex() if index < 0 { ap.log.Info(logs.AlphabetNonAlphabetModeIgnoreGasEmissionEvent) - return + return true } contract, ok := ap.alphabetContracts.GetByIndex(index) @@ -25,7 +26,7 @@ func (ap *Processor) processEmit() { ap.log.Debug(logs.AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent, zap.Int("index", index)) - return + return false } // there is no signature collecting, so we don't need extra fee @@ -33,13 +34,13 @@ func (ap *Processor) processEmit() { if err != nil { ap.log.Warn(logs.AlphabetCantInvokeAlphabetEmitMethod, zap.String("error", err.Error())) - return + return false } if ap.storageEmission == 0 { ap.log.Info(logs.AlphabetStorageNodeEmissionIsOff) - return + return true } networkMap, err := ap.netmapClient.NetMap() @@ -47,26 +48,31 @@ func (ap *Processor) processEmit() { ap.log.Warn(logs.AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes, zap.String("error", err.Error())) - return + return false } nmNodes := networkMap.Nodes() nmLen := len(nmNodes) - extraLen := len(ap.parsedWallets) + ap.pwLock.RLock() + pw := ap.parsedWallets + ap.pwLock.RUnlock() + extraLen := len(pw) ap.log.Debug(logs.AlphabetGasEmission, zap.Int("network_map", nmLen), zap.Int("extra_wallets", extraLen)) if nmLen+extraLen == 0 { - return + return true } gasPerNode := fixedn.Fixed8(ap.storageEmission / uint64(nmLen+extraLen)) ap.transferGasToNetmapNodes(nmNodes, gasPerNode) - ap.transferGasToExtraNodes(extraLen, gasPerNode) + ap.transferGasToExtraNodes(pw, gasPerNode) + + return true } func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerNode fixedn.Fixed8) { @@ -92,12 +98,12 @@ func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerN } } -func (ap *Processor) transferGasToExtraNodes(extraLen int, gasPerNode fixedn.Fixed8) { - if extraLen != 0 { - err := ap.morphClient.BatchTransferGas(ap.parsedWallets, gasPerNode) +func (ap *Processor) transferGasToExtraNodes(pw []util.Uint160, gasPerNode fixedn.Fixed8) { + if len(pw) > 0 { + err := ap.morphClient.BatchTransferGas(pw, gasPerNode) if err != nil { - receiversLog := make([]string, extraLen) - for i, addr := range ap.parsedWallets { + receiversLog := make([]string, len(pw)) + for i, addr := range pw { receiversLog[i] = addr.StringLE() } ap.log.Warn(logs.AlphabetCantTransferGasToWallet, diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index c2d7c1164..04dde80f7 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -3,9 +3,11 @@ package alphabet import ( "errors" "fmt" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -45,8 +47,11 @@ type ( // Processor of events produced for alphabet contracts in the sidechain. Processor struct { - parsedWallets []util.Uint160 + parsedWallets []util.Uint160 + // protects parsedWallets from concurrent change + pwLock sync.RWMutex log *logger.Logger + metrics metrics.Register pool *ants.Pool alphabetContracts Contracts netmapClient netmapClient @@ -59,6 +64,7 @@ type ( Params struct { ParsedWallets []util.Uint160 Log *logger.Logger + Metrics metrics.Register PoolSize int AlphabetContracts Contracts NetmapClient netmapClient @@ -86,9 +92,15 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/frostfs: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ parsedWallets: p.ParsedWallets, log: p.Log, + metrics: metricsRegister, pool: pool, alphabetContracts: p.AlphabetContracts, netmapClient: p.NetmapClient, @@ -98,6 +110,12 @@ func New(p *Params) (*Processor, error) { }, nil } +func (ap *Processor) SetParsedWallets(parsedWallets []util.Uint160) { + ap.pwLock.Lock() + ap.parsedWallets = parsedWallets + ap.pwLock.Unlock() +} + // ListenerNotificationParsers for the 'event.Listener' event producer. func (ap *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { return nil diff --git a/pkg/innerring/processors/balance/handlers.go b/pkg/innerring/processors/balance/handlers.go index e325da1f9..e39f3abbd 100644 --- a/pkg/innerring/processors/balance/handlers.go +++ b/pkg/innerring/processors/balance/handlers.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" "go.uber.org/zap" @@ -17,7 +18,9 @@ func (bp *Processor) handleLock(ev event.Event) { // send an event to the worker pool - err := bp.pool.Submit(func() { bp.processLock(&lock) }) + err := processors.SubmitEvent(bp.pool, bp.metrics, "lock", func() bool { + return bp.processLock(&lock) + }) if err != nil { // there system can be moved into controlled degradation stage bp.log.Warn(logs.BalanceBalanceWorkerPoolDrained, diff --git a/pkg/innerring/processors/balance/process_assets.go b/pkg/innerring/processors/balance/process_assets.go index 3f86a3cb7..1d94fa454 100644 --- a/pkg/innerring/processors/balance/process_assets.go +++ b/pkg/innerring/processors/balance/process_assets.go @@ -9,10 +9,10 @@ import ( // Process lock event by invoking Cheque method in main net to send assets // back to the withdraw issuer. -func (bp *Processor) processLock(lock *balanceEvent.Lock) { +func (bp *Processor) processLock(lock *balanceEvent.Lock) bool { if !bp.alphabetState.IsAlphabet() { bp.log.Info(logs.BalanceNonAlphabetModeIgnoreBalanceLock) - return + return true } prm := frostfsContract.ChequePrm{} @@ -26,5 +26,8 @@ func (bp *Processor) processLock(lock *balanceEvent.Lock) { err := bp.frostfsClient.Cheque(prm) if err != nil { bp.log.Error(logs.BalanceCantSendLockAssetTx, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/balance/processor.go b/pkg/innerring/processors/balance/processor.go index 356754cfb..5cc849b5c 100644 --- a/pkg/innerring/processors/balance/processor.go +++ b/pkg/innerring/processors/balance/processor.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" @@ -32,6 +33,7 @@ type ( // Processor of events produced by balance contract in the morphchain. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool frostfsClient FrostFSClient balanceSC util.Uint160 @@ -42,6 +44,7 @@ type ( // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int FrostFSClient FrostFSClient BalanceSC util.Uint160 @@ -72,8 +75,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/balance: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, frostfsClient: p.FrostFSClient, balanceSC: p.BalanceSC, diff --git a/pkg/innerring/processors/container/common.go b/pkg/innerring/processors/container/common.go index 375e4c179..97eb6f559 100644 --- a/pkg/innerring/processors/container/common.go +++ b/pkg/innerring/processors/container/common.go @@ -63,7 +63,6 @@ func (cp *Processor) verifySignature(v signatureVerificationData) error { } if keyProvided { - // TODO(@cthulhu-rider): #1387 use another approach after neofs-sdk-go#233 var idFromKey user.ID user.IDFromKey(&idFromKey, (ecdsa.PublicKey)(key)) @@ -118,8 +117,6 @@ func (cp *Processor) verifyByTokenSession(v signatureVerificationData, key *fros return errors.New("invalid session token signature") } - // FIXME(@cthulhu-rider): #1387 check token is signed by container owner, see neofs-sdk-go#233 - if keyProvided && !tok.AssertAuthKey(key) { return errors.New("signed with a non-session key") } diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index 2ab1147c8..3ec10b889 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "github.com/mr-tron/base58" @@ -20,7 +21,9 @@ func (cp *Processor) handlePut(ev event.Event) { // send an event to the worker pool - err := cp.pool.Submit(func() { cp.processContainerPut(put) }) + err := processors.SubmitEvent(cp.pool, cp.metrics, "container_put", func() bool { + return cp.processContainerPut(put) + }) if err != nil { // there system can be moved into controlled degradation stage cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, @@ -36,7 +39,9 @@ func (cp *Processor) handleDelete(ev event.Event) { // send an event to the worker pool - err := cp.pool.Submit(func() { cp.processContainerDelete(del) }) + err := processors.SubmitEvent(cp.pool, cp.metrics, "container_delete", func() bool { + return cp.processContainerDelete(del) + }) if err != nil { // there system can be moved into controlled degradation stage cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, @@ -53,8 +58,8 @@ func (cp *Processor) handleSetEACL(ev event.Event) { // send an event to the worker pool - err := cp.pool.Submit(func() { - cp.processSetEACL(e) + err := processors.SubmitEvent(cp.pool, cp.metrics, "container_set_eacl", func() bool { + return cp.processSetEACL(e) }) if err != nil { // there system can be moved into controlled degradation stage diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index 6075ff577..1e518f474 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -7,7 +7,6 @@ import ( "time" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" @@ -22,6 +21,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/network/payload" "github.com/nspcc-dev/neo-go/pkg/util" @@ -34,18 +34,16 @@ func TestPutEvent(t *testing.T) { homHashDisabled: true, epoch: 100, } - cc := &testContainerClient{ - get: make(map[string]*containercore.Container), - } + mc := &testMorphClient{} proc, err := New(&Params{ Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, FrostFSIDClient: &testIDClient{}, - NotaryDisabled: true, NetworkState: nst, - ContainerClient: cc, + ContainerClient: &testContainerClient{}, + MorphClient: mc, }) require.NoError(t, err, "failed to create processor") @@ -64,10 +62,15 @@ func TestPutEvent(t *testing.T) { cnr.SetBasicACL(acl.Private) containerSDK.DisableHomomorphicHashing(&cnr) + nr := &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + } + event := &testPutEvent{ cnr: &cnr, pk: p, st: nil, + nr: nr, } proc.handlePut(event) @@ -76,13 +79,7 @@ func TestPutEvent(t *testing.T) { time.Sleep(10 * time.Millisecond) } - var expectedPut cntClient.PutPrm - expectedPut.SetContainer(cnr.Marshal()) - expectedPut.SetKey(p.PublicKey().Bytes()) - expectedPut.SetSignature(p.Sign(cnr.Marshal())) - expectedPut.SetZone("container") - - require.EqualValues(t, []cntClient.PutPrm{expectedPut}, cc.put, "invalid put requests") + require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") } func TestDeleteEvent(t *testing.T) { @@ -103,15 +100,16 @@ func TestDeleteEvent(t *testing.T) { p.PublicKey(), }, } + mc := &testMorphClient{} proc, err := New(&Params{ Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, FrostFSIDClient: idc, - NotaryDisabled: true, NetworkState: nst, ContainerClient: cc, + MorphClient: mc, }) require.NoError(t, err, "failed to create processor") @@ -133,9 +131,14 @@ func TestDeleteEvent(t *testing.T) { cidBin := make([]byte, 32) cid.Encode(cidBin) + nr := &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + } + ev := containerEvent.Delete{ - ContainerIDValue: cidBin, - SignatureValue: p.Sign(cidBin), + ContainerIDValue: cidBin, + SignatureValue: p.Sign(cidBin), + NotaryRequestValue: nr, } var signature frostfscrypto.Signature @@ -156,7 +159,7 @@ func TestDeleteEvent(t *testing.T) { expectedDelete.SetCID(ev.ContainerID()) expectedDelete.SetSignature(ev.Signature()) - require.EqualValues(t, []cntClient.DeletePrm{expectedDelete}, cc.delete, "invalid delete requests") + require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") } func TestSetEACLEvent(t *testing.T) { @@ -168,15 +171,16 @@ func TestSetEACLEvent(t *testing.T) { cc := &testContainerClient{ get: make(map[string]*containercore.Container), } + mc := &testMorphClient{} proc, err := New(&Params{ Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, FrostFSIDClient: &testIDClient{}, - NotaryDisabled: true, NetworkState: nst, ContainerClient: cc, + MorphClient: mc, }) require.NoError(t, err, "failed to create processor") @@ -219,10 +223,14 @@ func TestSetEACLEvent(t *testing.T) { table.AddRecord(r) + nr := &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + } event := containerEvent.SetEACL{ - TableValue: table.ToV2().StableMarshal(nil), - PublicKeyValue: p.PublicKey().Bytes(), - SignatureValue: p.Sign(table.ToV2().StableMarshal(nil)), + TableValue: table.ToV2().StableMarshal(nil), + PublicKeyValue: p.PublicKey().Bytes(), + SignatureValue: p.Sign(table.ToV2().StableMarshal(nil)), + NotaryRequestValue: nr, } proc.handleSetEACL(event) @@ -236,7 +244,7 @@ func TestSetEACLEvent(t *testing.T) { expectedPutEACL.SetKey(p.PublicKey().Bytes()) expectedPutEACL.SetSignature(p.Sign(table.ToV2().StableMarshal(nil))) - require.EqualValues(t, []cntClient.PutEACLPrm{expectedPutEACL}, cc.putEACL, "invalid set EACL requests") + require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") } type testAlphabetState struct { @@ -262,25 +270,13 @@ func (s *testNetworkState) Epoch() (uint64, error) { type testContainerClient struct { contractAddress util.Uint160 - put []cntClient.PutPrm get map[string]*containercore.Container - delete []cntClient.DeletePrm - putEACL []cntClient.PutEACLPrm } func (c *testContainerClient) ContractAddress() util.Uint160 { return c.contractAddress } -func (c *testContainerClient) Morph() *client.Client { - return nil -} - -func (c *testContainerClient) Put(p cntClient.PutPrm) error { - c.put = append(c.put, p) - return nil -} - func (c *testContainerClient) Get(cid []byte) (*containercore.Container, error) { key := hex.EncodeToString(cid) if cont, found := c.get[key]; found { @@ -289,16 +285,6 @@ func (c *testContainerClient) Get(cid []byte) (*containercore.Container, error) return nil, apistatus.ContainerNotFound{} } -func (c *testContainerClient) Delete(p cntClient.DeletePrm) error { - c.delete = append(c.delete, p) - return nil -} - -func (c *testContainerClient) PutEACL(p cntClient.PutEACLPrm) error { - c.putEACL = append(c.putEACL, p) - return nil -} - type testIDClient struct { publicKeys keys.PublicKeys } @@ -313,6 +299,7 @@ type testPutEvent struct { cnr *containerSDK.Container pk *keys.PrivateKey st []byte + nr *payload.P2PNotaryRequest } func (e *testPutEvent) MorphEvent() {} @@ -333,5 +320,14 @@ func (e *testPutEvent) SessionToken() []byte { return e.st } func (e *testPutEvent) NotaryRequest() *payload.P2PNotaryRequest { + return e.nr +} + +type testMorphClient struct { + transactions []*transaction.Transaction +} + +func (c *testMorphClient) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { + c.transactions = append(c.transactions, mainTx) return nil } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 603ab86d1..2629b9d29 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -4,7 +4,6 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -32,10 +31,10 @@ type putContainerContext struct { // Process a new container from the user by checking the container sanity // and sending approve tx back to the morph. -func (cp *Processor) processContainerPut(put putEvent) { +func (cp *Processor) processContainerPut(put putEvent) bool { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerPut) - return + return true } ctx := &putContainerContext{ @@ -48,10 +47,17 @@ func (cp *Processor) processContainerPut(put putEvent) { zap.String("error", err.Error()), ) - return + return false } - cp.approvePutContainer(ctx) + if err := cp.morphClient.NotarySignAndInvokeTX(ctx.e.NotaryRequest().MainTransaction); err != nil { + cp.log.Error(logs.ContainerCouldNotApprovePutContainer, + zap.String("error", err.Error()), + ) + return false + } + + return true } func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { @@ -90,40 +96,12 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { return nil } -func (cp *Processor) approvePutContainer(ctx *putContainerContext) { - e := ctx.e - - var err error - - prm := cntClient.PutPrm{} - - prm.SetContainer(e.Container()) - prm.SetKey(e.PublicKey()) - prm.SetSignature(e.Signature()) - prm.SetToken(e.SessionToken()) - prm.SetName(ctx.d.Name()) - prm.SetZone(ctx.d.Zone()) - - if nr := e.NotaryRequest(); nr != nil { - // put event was received via Notary service - err = cp.cnrClient.Morph().NotarySignAndInvokeTX(nr.MainTransaction) - } else { - // put event was received via notification service - err = cp.cnrClient.Put(prm) - } - if err != nil { - cp.log.Error(logs.ContainerCouldNotApprovePutContainer, - zap.String("error", err.Error()), - ) - } -} - // Process delete container operation from the user by checking container sanity // and sending approve tx back to morph. -func (cp *Processor) processContainerDelete(e containerEvent.Delete) { +func (cp *Processor) processContainerDelete(e containerEvent.Delete) bool { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerDelete) - return + return true } err := cp.checkDeleteContainer(e) @@ -132,10 +110,18 @@ func (cp *Processor) processContainerDelete(e containerEvent.Delete) { zap.String("error", err.Error()), ) - return + return false } - cp.approveDeleteContainer(e) + if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { + cp.log.Error(logs.ContainerCouldNotApproveDeleteContainer, + zap.String("error", err.Error()), + ) + + return false + } + + return true } func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { @@ -162,6 +148,7 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { binTokenSession: e.SessionToken(), signature: e.Signature(), signedData: binCnr, + binPublicKey: e.PublicKeyValue, }) if err != nil { return fmt.Errorf("auth container removal: %w", err) @@ -170,29 +157,6 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { return nil } -func (cp *Processor) approveDeleteContainer(e containerEvent.Delete) { - var err error - - prm := cntClient.DeletePrm{} - - prm.SetCID(e.ContainerID()) - prm.SetSignature(e.Signature()) - prm.SetToken(e.SessionToken()) - - if nr := e.NotaryRequest(); nr != nil { - // delete event was received via Notary service - err = cp.cnrClient.Morph().NotarySignAndInvokeTX(nr.MainTransaction) - } else { - // delete event was received via notification service - err = cp.cnrClient.Delete(prm) - } - if err != nil { - cp.log.Error(logs.ContainerCouldNotApproveDeleteContainer, - zap.String("error", err.Error()), - ) - } -} - func checkNNS(ctx *putContainerContext, cnr containerSDK.Container) error { // fetch domain info ctx.d = containerSDK.ReadDomain(cnr) diff --git a/pkg/innerring/processors/container/process_eacl.go b/pkg/innerring/processors/container/process_eacl.go index a8d880c8f..8ab0d5c39 100644 --- a/pkg/innerring/processors/container/process_eacl.go +++ b/pkg/innerring/processors/container/process_eacl.go @@ -12,10 +12,10 @@ import ( "go.uber.org/zap" ) -func (cp *Processor) processSetEACL(e containerEvent.SetEACL) { +func (cp *Processor) processSetEACL(e containerEvent.SetEACL) bool { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreSetEACL) - return + return true } err := cp.checkSetEACL(e) @@ -24,10 +24,17 @@ func (cp *Processor) processSetEACL(e containerEvent.SetEACL) { zap.String("error", err.Error()), ) - return + return false } - cp.approveSetEACL(e) + if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { + cp.log.Error(logs.ContainerCouldNotApproveSetEACL, + zap.String("error", err.Error()), + ) + return false + } + + return true } func (cp *Processor) checkSetEACL(e containerEvent.SetEACL) error { @@ -73,27 +80,3 @@ func (cp *Processor) checkSetEACL(e containerEvent.SetEACL) error { return nil } - -func (cp *Processor) approveSetEACL(e containerEvent.SetEACL) { - var err error - - prm := cntClient.PutEACLPrm{} - - prm.SetTable(e.Table()) - prm.SetKey(e.PublicKey()) - prm.SetSignature(e.Signature()) - prm.SetToken(e.SessionToken()) - - if nr := e.NotaryRequest(); nr != nil { - // setEACL event was received via Notary service - err = cp.cnrClient.Morph().NotarySignAndInvokeTX(nr.MainTransaction) - } else { - // setEACL event was received via notification service - err = cp.cnrClient.PutEACL(prm) - } - if err != nil { - cp.log.Error(logs.ContainerCouldNotApproveSetEACL, - zap.String("error", err.Error()), - ) - } -} diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index d5af5e394..fd5348c6f 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -6,13 +6,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/core/mempoolevent" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" @@ -27,11 +27,11 @@ type ( ContClient interface { ContractAddress() util.Uint160 - Morph() *client.Client - Put(p cntClient.PutPrm) error Get(cid []byte) (*containercore.Container, error) - Delete(p cntClient.DeletePrm) error - PutEACL(p cntClient.PutEACLPrm) error + } + + MorphClient interface { + NotarySignAndInvokeTX(mainTx *transaction.Transaction) error } IDClient interface { @@ -40,24 +40,26 @@ type ( // Processor of events produced by container contract in the sidechain. Processor struct { - log *logger.Logger - pool *ants.Pool - alphabetState AlphabetState - cnrClient ContClient // notary must be enabled - idClient IDClient - netState NetworkState - notaryDisabled bool + log *logger.Logger + metrics metrics.Register + pool *ants.Pool + alphabetState AlphabetState + cnrClient ContClient // notary must be enabled + morphClient MorphClient + idClient IDClient + netState NetworkState } // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int AlphabetState AlphabetState ContainerClient ContClient + MorphClient MorphClient FrostFSIDClient IDClient NetworkState NetworkState - NotaryDisabled bool } ) @@ -79,13 +81,6 @@ type NetworkState interface { HomomorphicHashDisabled() (bool, error) } -const ( - putNotification = "containerPut" - deleteNotification = "containerDelete" - - setEACLNotification = "setEACL" -) - // New creates a container contract processor instance. func New(p *Params) (*Processor, error) { switch { @@ -95,6 +90,8 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/container: global state is not set") case p.ContainerClient == nil: return nil, errors.New("ir/container: Container client is not set") + case p.MorphClient == nil: + return nil, errors.New("ir/container: Morph client is not set") case p.FrostFSIDClient == nil: return nil, errors.New("ir/container: FrostFS ID client is not set") case p.NetworkState == nil: @@ -108,79 +105,31 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/container: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ - log: p.Log, - pool: pool, - alphabetState: p.AlphabetState, - cnrClient: p.ContainerClient, - idClient: p.FrostFSIDClient, - netState: p.NetworkState, - notaryDisabled: p.NotaryDisabled, + log: p.Log, + metrics: metricsRegister, + pool: pool, + alphabetState: p.AlphabetState, + cnrClient: p.ContainerClient, + idClient: p.FrostFSIDClient, + netState: p.NetworkState, + morphClient: p.MorphClient, }, nil } // ListenerNotificationParsers for the 'event.Listener' event producer. func (cp *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - if !cp.notaryDisabled { - return nil - } - - var ( - parsers = make([]event.NotificationParserInfo, 0, 3) - - p event.NotificationParserInfo - ) - - p.SetScriptHash(cp.cnrClient.ContractAddress()) - - // container put - p.SetType(event.TypeFromString(putNotification)) - p.SetParser(containerEvent.ParsePut) - parsers = append(parsers, p) - - // container delete - p.SetType(event.TypeFromString(deleteNotification)) - p.SetParser(containerEvent.ParseDelete) - parsers = append(parsers, p) - - // set eACL - p.SetType(event.TypeFromString(setEACLNotification)) - p.SetParser(containerEvent.ParseSetEACL) - parsers = append(parsers, p) - - return parsers + return nil } // ListenerNotificationHandlers for the 'event.Listener' event producer. func (cp *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { - if !cp.notaryDisabled { - return nil - } - - var ( - handlers = make([]event.NotificationHandlerInfo, 0, 3) - - h event.NotificationHandlerInfo - ) - - h.SetScriptHash(cp.cnrClient.ContractAddress()) - - // container put - h.SetType(event.TypeFromString(putNotification)) - h.SetHandler(cp.handlePut) - handlers = append(handlers, h) - - // container delete - h.SetType(event.TypeFromString(deleteNotification)) - h.SetHandler(cp.handleDelete) - handlers = append(handlers, h) - - // set eACL - h.SetType(event.TypeFromString(setEACLNotification)) - h.SetHandler(cp.handleSetEACL) - handlers = append(handlers, h) - - return handlers + return nil } // ListenerNotaryParsers for the 'event.Listener' notary event producer. diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index 574cf057a..ab53d5c48 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -18,7 +19,9 @@ func (np *Processor) handleDeposit(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processDeposit(deposit) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_deposit", func() bool { + return np.processDeposit(deposit) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -34,7 +37,9 @@ func (np *Processor) handleWithdraw(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processWithdraw(withdraw) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_withdraw", func() bool { + return np.processWithdraw(withdraw) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -50,7 +55,9 @@ func (np *Processor) handleCheque(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processCheque(cheque) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_cheque", func() bool { + return np.processCheque(cheque) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -67,7 +74,9 @@ func (np *Processor) handleConfig(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processConfig(cfg) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_config", func() bool { + return np.processConfig(cfg) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -83,7 +92,9 @@ func (np *Processor) handleBind(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processBind(e, true) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_bind", func() bool { + return np.processBind(e, true) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -99,7 +110,9 @@ func (np *Processor) handleUnbind(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processBind(e, false) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_unbind", func() bool { + return np.processBind(e, false) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, diff --git a/pkg/innerring/processors/frostfs/process_assets.go b/pkg/innerring/processors/frostfs/process_assets.go index cfbf21b08..327a4a3aa 100644 --- a/pkg/innerring/processors/frostfs/process_assets.go +++ b/pkg/innerring/processors/frostfs/process_assets.go @@ -15,10 +15,10 @@ const ( // Process deposit event by invoking a balance contract and sending native // gas in the sidechain. -func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { +func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreDeposit) - return + return true } prm := balance.MintPrm{} @@ -49,7 +49,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { zap.Uint64("last_emission", val), zap.Uint64("current_epoch", curEpoch)) - return + return false } // get gas balance of the node @@ -57,7 +57,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { balance, err := np.morphClient.GasBalance() if err != nil { np.log.Error(logs.FrostFSCantGetGasBalanceOfTheNode, zap.Error(err)) - return + return false } if balance < np.gasBalanceThreshold { @@ -65,7 +65,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { zap.Int64("balance", balance), zap.Int64("threshold", np.gasBalanceThreshold)) - return + return false } err = np.morphClient.TransferGas(receiver, np.mintEmitValue) @@ -73,24 +73,26 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { np.log.Error(logs.FrostFSCantTransferNativeGasToReceiver, zap.String("error", err.Error())) - return + return false } np.mintEmitCache.Add(receiver.String(), curEpoch) + + return true } // Process withdraw event by locking assets in the balance account. -func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) { +func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreWithdraw) - return + return true } // create lock account lock, err := util.Uint160DecodeBytesBE(withdraw.ID()[:util.Uint160Size]) if err != nil { np.log.Error(logs.FrostFSCantCreateLockAccount, zap.Error(err)) - return + return false } curEpoch := np.epochState.EpochCounter() @@ -106,15 +108,18 @@ func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) { err = np.balanceClient.Lock(prm) if err != nil { np.log.Error(logs.FrostFSCantLockAssetsForWithdraw, zap.Error(err)) + return false } + + return true } // Process cheque event by transferring assets from the lock account back to // the reserve account. -func (np *Processor) processCheque(cheque frostfsEvent.Cheque) { +func (np *Processor) processCheque(cheque frostfsEvent.Cheque) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreCheque) - return + return true } prm := balance.BurnPrm{} @@ -126,5 +131,8 @@ func (np *Processor) processCheque(cheque frostfsEvent.Cheque) { err := np.balanceClient.Burn(prm) if err != nil { np.log.Error(logs.FrostFSCantTransferAssetsToFedContract, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/frostfs/process_bind.go b/pkg/innerring/processors/frostfs/process_bind.go index a9b523a77..50c6bf5f5 100644 --- a/pkg/innerring/processors/frostfs/process_bind.go +++ b/pkg/innerring/processors/frostfs/process_bind.go @@ -18,10 +18,10 @@ type bindCommon interface { TxHash() util.Uint256 } -func (np *Processor) processBind(e bindCommon, bind bool) { +func (np *Processor) processBind(e bindCommon, bind bool) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreBind) - return + return true } c := &bindCommonContext{ @@ -36,10 +36,10 @@ func (np *Processor) processBind(e bindCommon, bind bool) { zap.String("error", err.Error()), ) - return + return false } - np.approveBindCommon(c) + return np.approveBindCommon(c) == nil } type bindCommonContext struct { @@ -70,7 +70,7 @@ func (np *Processor) checkBindCommon(e *bindCommonContext) error { return nil } -func (np *Processor) approveBindCommon(e *bindCommonContext) { +func (np *Processor) approveBindCommon(e *bindCommonContext) error { // calculate wallet address scriptHash := e.User() @@ -80,7 +80,7 @@ func (np *Processor) approveBindCommon(e *bindCommonContext) { zap.String("error", err.Error()), ) - return + return err } var id user.ID @@ -104,4 +104,6 @@ func (np *Processor) approveBindCommon(e *bindCommonContext) { np.log.Error(fmt.Sprintf("could not approve %s", typ), zap.String("error", err.Error())) } + + return err } diff --git a/pkg/innerring/processors/frostfs/process_config.go b/pkg/innerring/processors/frostfs/process_config.go index ce2dabfdd..2ae3e6ced 100644 --- a/pkg/innerring/processors/frostfs/process_config.go +++ b/pkg/innerring/processors/frostfs/process_config.go @@ -9,10 +9,10 @@ import ( // Process config event by setting configuration value from the mainchain in // the sidechain. -func (np *Processor) processConfig(config frostfsEvent.Config) { +func (np *Processor) processConfig(config frostfsEvent.Config) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreConfig) - return + return true } prm := nmClient.SetConfigPrm{} @@ -25,5 +25,8 @@ func (np *Processor) processConfig(config frostfsEvent.Config) { err := np.netmapClient.SetConfig(prm) if err != nil { np.log.Error(logs.FrostFSCantRelaySetConfigEvent, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index 2af15a814..20f44adcd 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -6,6 +6,7 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -58,6 +59,7 @@ type ( // Processor of events produced by frostfs contract in main net. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool frostfsContract util.Uint160 balanceClient BalanceClient @@ -66,7 +68,7 @@ type ( epochState EpochState alphabetState AlphabetState converter PrecisionConverter - mintEmitLock *sync.Mutex + mintEmitLock sync.Mutex mintEmitCache *lru.Cache[string, uint64] mintEmitThreshold uint64 mintEmitValue fixedn.Fixed8 @@ -77,6 +79,7 @@ type ( // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int FrostFSContract util.Uint160 FrostFSIDClient IDClient @@ -129,8 +132,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/frostfs: can't create LRU cache for gas emission: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, frostfsContract: p.FrostFSContract, balanceClient: p.BalanceClient, @@ -139,7 +148,6 @@ func New(p *Params) (*Processor, error) { epochState: p.EpochState, alphabetState: p.AlphabetState, converter: p.Converter, - mintEmitLock: new(sync.Mutex), mintEmitCache: lruCache, mintEmitThreshold: p.MintEmitThreshold, mintEmitValue: p.MintEmitValue, diff --git a/pkg/innerring/processors/governance/handlers.go b/pkg/innerring/processors/governance/handlers.go index 727acc21a..fd7f539c3 100644 --- a/pkg/innerring/processors/governance/handlers.go +++ b/pkg/innerring/processors/governance/handlers.go @@ -2,6 +2,7 @@ package governance import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/rolemanagement" "github.com/nspcc-dev/neo-go/pkg/core/native" @@ -35,7 +36,9 @@ func (gp *Processor) HandleAlphabetSync(e event.Event) { // send event to the worker pool - err := gp.pool.Submit(func() { gp.processAlphabetSync(hash) }) + err := processors.SubmitEvent(gp.pool, gp.metrics, "alphabet_sync", func() bool { + return gp.processAlphabetSync(hash) + }) if err != nil { // there system can be moved into controlled degradation stage gp.log.Warn(logs.GovernanceGovernanceWorkerPoolDrained, diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go index 23f91869a..63d156dac 100644 --- a/pkg/innerring/processors/governance/handlers_test.go +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -42,16 +42,15 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { proc, err := New( &Params{ - Log: test.NewLogger(t, true), - EpochState: es, - AlphabetState: as, - Voter: v, - IRFetcher: irf, - NotaryDisabled: true, - MorphClient: m, - MainnetClient: mn, - FrostFSClient: f, - NetmapClient: nm, + Log: test.NewLogger(t, true), + EpochState: es, + AlphabetState: as, + Voter: v, + IRFetcher: irf, + MorphClient: m, + MainnetClient: mn, + FrostFSClient: f, + NetmapClient: nm, }, ) @@ -74,17 +73,19 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { }, }, v.votes, "invalid vote calls") - var irUpdateExp nmClient.UpdateIRPrm - irUpdateExp.SetKeys(testKeys.newInnerRingExp) - irUpdateExp.SetHash(ev.txHash) + var irUpdateExp []nmClient.UpdateIRPrm - require.EqualValues(t, []nmClient.UpdateIRPrm{irUpdateExp}, nm.updates, "invalid IR updates") + require.EqualValues(t, irUpdateExp, nm.updates, "invalid IR updates") - var expAlphabetUpdates []client.UpdateAlphabetListPrm - require.EqualValues(t, expAlphabetUpdates, m.alphabetUpdates, "invalid alphabet updates") + var expAlphabetUpdate client.UpdateAlphabetListPrm + expAlphabetUpdate.SetHash(ev.txHash) + expAlphabetUpdate.SetList(testKeys.newInnerRingExp) + require.EqualValues(t, []client.UpdateAlphabetListPrm{expAlphabetUpdate}, m.alphabetUpdates, "invalid alphabet updates") - var expNotaryUpdates []client.UpdateNotaryListPrm - require.EqualValues(t, expNotaryUpdates, m.notaryUpdates, "invalid notary list updates") + var expNotaryUpdate client.UpdateNotaryListPrm + expNotaryUpdate.SetHash(ev.txHash) + expNotaryUpdate.SetList(testKeys.newAlphabetExp) + require.EqualValues(t, []client.UpdateNotaryListPrm{expNotaryUpdate}, m.notaryUpdates, "invalid notary list updates") buf := make([]byte, 8) binary.LittleEndian.PutUint64(buf, es.epoch) @@ -122,16 +123,15 @@ func TestHandleAlphabetDesignateEvent(t *testing.T) { proc, err := New( &Params{ - Log: test.NewLogger(t, true), - EpochState: es, - AlphabetState: as, - Voter: v, - IRFetcher: irf, - NotaryDisabled: false, - MorphClient: m, - MainnetClient: mn, - FrostFSClient: f, - NetmapClient: nm, + Log: test.NewLogger(t, true), + EpochState: es, + AlphabetState: as, + Voter: v, + IRFetcher: irf, + MorphClient: m, + MainnetClient: mn, + FrostFSClient: f, + NetmapClient: nm, }, ) diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index 629d8741e..50ba58e77 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" - nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" @@ -19,36 +18,36 @@ const ( alphabetUpdateIDPrefix = "AlphabetUpdate" ) -func (gp *Processor) processAlphabetSync(txHash util.Uint256) { +func (gp *Processor) processAlphabetSync(txHash util.Uint256) bool { if !gp.alphabetState.IsAlphabet() { gp.log.Info(logs.GovernanceNonAlphabetModeIgnoreAlphabetSync) - return + return true } mainnetAlphabet, err := gp.mainnetClient.NeoFSAlphabetList() if err != nil { gp.log.Error(logs.GovernanceCantFetchAlphabetListFromMainNet, zap.String("error", err.Error())) - return + return false } sidechainAlphabet, err := gp.morphClient.Committee() if err != nil { gp.log.Error(logs.GovernanceCantFetchAlphabetListFromSideChain, zap.String("error", err.Error())) - return + return false } newAlphabet, err := newAlphabetList(sidechainAlphabet, mainnetAlphabet) if err != nil { gp.log.Error(logs.GovernanceCantMergeAlphabetListsFromMainNetAndSideChain, zap.String("error", err.Error())) - return + return false } if newAlphabet == nil { gp.log.Info(logs.GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged) - return + return true } gp.log.Info(logs.GovernanceAlphabetListHasBeenChangedStartingUpdate, @@ -78,6 +77,8 @@ func (gp *Processor) processAlphabetSync(txHash util.Uint256) { gp.updateFrostFSContractInMainnet(newAlphabet) gp.log.Info(logs.GovernanceFinishedAlphabetListUpdate) + + return true } func prettyKeys(keys keys.PublicKeys) string { @@ -114,33 +115,17 @@ func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(sidechainAlphabet, newAl zap.String("after", prettyKeys(newInnerRing)), ) - if gp.notaryDisabled { - updPrm := nmClient.UpdateIRPrm{} + updPrm := client.UpdateAlphabetListPrm{} + updPrm.SetList(newInnerRing) + updPrm.SetHash(txHash) - updPrm.SetKeys(newInnerRing) - updPrm.SetHash(txHash) - - err = gp.netmapClient.UpdateInnerRing(updPrm) - } else { - updPrm := client.UpdateAlphabetListPrm{} - - updPrm.SetList(newInnerRing) - updPrm.SetHash(txHash) - - err = gp.morphClient.UpdateNeoFSAlphabetList(updPrm) - } - - if err != nil { + if err = gp.morphClient.UpdateNeoFSAlphabetList(updPrm); err != nil { gp.log.Error(logs.GovernanceCantUpdateInnerRingListWithNewAlphabetKeys, zap.String("error", err.Error())) } } func (gp *Processor) updateNotaryRoleInSidechain(newAlphabet keys.PublicKeys, txHash util.Uint256) { - if gp.notaryDisabled { - return - } - updPrm := client.UpdateNotaryListPrm{} updPrm.SetList(newAlphabet) diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index e08bd3809..fa267eade 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -75,6 +76,7 @@ type ( // Processor of events related to governance in the network. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool frostfsClient FrostFSClient netmapClient NetmapClient @@ -87,14 +89,13 @@ type ( mainnetClient MainnetClient morphClient MorphClient - notaryDisabled bool - designate util.Uint160 } // Params of the processor constructor. Params struct { - Log *logger.Logger + Log *logger.Logger + Metrics metrics.Register AlphabetState AlphabetState EpochState EpochState @@ -105,8 +106,6 @@ type ( MainnetClient MainnetClient FrostFSClient FrostFSClient NetmapClient NetmapClient - - NotaryDisabled bool } ) @@ -134,22 +133,27 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/governance: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + // result is cached by neo-go, so we can pre-calc it designate := p.MainnetClient.GetDesignateHash() return &Processor{ - log: p.Log, - pool: pool, - frostfsClient: p.FrostFSClient, - netmapClient: p.NetmapClient, - alphabetState: p.AlphabetState, - epochState: p.EpochState, - voter: p.Voter, - irFetcher: p.IRFetcher, - mainnetClient: p.MainnetClient, - morphClient: p.MorphClient, - notaryDisabled: p.NotaryDisabled, - designate: designate, + log: p.Log, + metrics: metricsRegister, + pool: pool, + frostfsClient: p.FrostFSClient, + netmapClient: p.NetmapClient, + alphabetState: p.AlphabetState, + epochState: p.EpochState, + voter: p.Voter, + irFetcher: p.IRFetcher, + mainnetClient: p.MainnetClient, + morphClient: p.MorphClient, + designate: designate, }, nil } diff --git a/pkg/innerring/processors/netmap/cleanup_table.go b/pkg/innerring/processors/netmap/cleanup_table.go index e4024e95f..80117247d 100644 --- a/pkg/innerring/processors/netmap/cleanup_table.go +++ b/pkg/innerring/processors/netmap/cleanup_table.go @@ -9,7 +9,7 @@ import ( type ( cleanupTable struct { - *sync.RWMutex + sync.RWMutex enabled bool threshold uint64 lastAccess map[string]epochStampWithNodeInfo @@ -29,7 +29,6 @@ type ( func newCleanupTable(enabled bool, threshold uint64) cleanupTable { return cleanupTable{ - RWMutex: new(sync.RWMutex), enabled: enabled, threshold: threshold, lastAccess: make(map[string]epochStampWithNodeInfo), diff --git a/pkg/innerring/processors/netmap/handlers.go b/pkg/innerring/processors/netmap/handlers.go index 6adeac562..c6053e281 100644 --- a/pkg/innerring/processors/netmap/handlers.go +++ b/pkg/innerring/processors/netmap/handlers.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -16,7 +17,7 @@ func (np *Processor) HandleNewEpochTick(ev event.Event) { // send an event to the worker pool - err := np.pool.Submit(func() { np.processNewEpochTick() }) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch_tick", np.processNewEpochTick) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, @@ -32,8 +33,8 @@ func (np *Processor) handleNewEpoch(ev event.Event) { // send an event to the worker pool - err := np.pool.Submit(func() { - np.processNewEpoch(epochEvent) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch", func() bool { + return np.processNewEpoch(epochEvent) }) if err != nil { // there system can be moved into controlled degradation stage @@ -51,8 +52,8 @@ func (np *Processor) handleAddPeer(ev event.Event) { // send an event to the worker pool - err := np.pool.Submit(func() { - np.processAddPeer(newPeer) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_add_peer", func() bool { + return np.processAddPeer(newPeer) }) if err != nil { // there system can be moved into controlled degradation stage @@ -69,8 +70,8 @@ func (np *Processor) handleUpdateState(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { - np.processUpdatePeer(updPeer) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_update_peer", func() bool { + return np.processUpdatePeer(updPeer) }) if err != nil { // there system can be moved into controlled degradation stage @@ -91,8 +92,8 @@ func (np *Processor) handleCleanupTick(ev event.Event) { np.log.Info(logs.NetmapTick, zap.String("type", "netmap cleaner")) // send event to the worker pool - err := np.pool.Submit(func() { - np.processNetmapCleanupTick(cleanup) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_cleanup_tick", func() bool { + return np.processNetmapCleanupTick(cleanup) }) if err != nil { // there system can be moved into controlled degradation stage diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 4905b45d2..6c9e265cc 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -31,7 +31,6 @@ func TestNewEpochTick(t *testing.T) { nc := &testNetmapClient{} proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true p.CleanupEnabled = true p.EpochState = es p.NetmapClient = nc @@ -80,7 +79,6 @@ func TestNewEpoch(t *testing.T) { eh := &testEventHandler{} proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true p.NotaryDepositHandler = eh.Handle p.AlphabetSyncHandler = eh.Handle p.NetmapClient = nc @@ -119,276 +117,139 @@ func TestNewEpoch(t *testing.T) { func TestAddPeer(t *testing.T) { t.Parallel() - t.Run("with notary", func(t *testing.T) { - t.Parallel() - nc := &testNetmapClient{ - contractAddress: util.Uint160{47}, - } + nc := &testNetmapClient{ + contractAddress: util.Uint160{47}, + } - proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true - p.NetmapClient = nc - }) - - require.NoError(t, err, "failed to create processor") - - var node netmap.NodeInfo - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key1") - node.SetPublicKey(key.Bytes()) - - ev := netmapEvent.AddPeer{ - NodeBytes: node.Marshal(), - Request: &payload.P2PNotaryRequest{ - MainTransaction: &transaction.Transaction{ - Nonce: 100, - }, - }, - } - proc.handleAddPeer(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - require.EqualValues(t, []notaryInvoke{ - { - contract: nc.contractAddress, - fee: 0, - nonce: ev.Request.MainTransaction.Nonce, - vub: nil, - method: "addPeerIR", - args: []any{ev.Node()}, - }, - }, nc.notaryInvokes, "invalid notary invokes") + proc, err := newTestProc(t, func(p *Params) { + p.NetmapClient = nc }) - t.Run("without notary", func(t *testing.T) { - t.Parallel() + require.NoError(t, err, "failed to create processor") - nc := &testNetmapClient{ - contractAddress: util.Uint160{47}, - } + var node netmap.NodeInfo + key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key") + node.SetPublicKey(key.Bytes()) - proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true - p.NetmapClient = nc - }) + ev := netmapEvent.AddPeer{ + NodeBytes: node.Marshal(), + Request: &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + }, + } + proc.handleAddPeer(ev) - require.NoError(t, err, "failed to create processor") + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } - var node netmap.NodeInfo - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key") - node.SetPublicKey(key.Bytes()) - - ev := netmapEvent.AddPeer{ - NodeBytes: node.Marshal(), - } - proc.handleAddPeer(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - var addPeerExp netmapclient.AddPeerPrm - addPeerExp.SetNodeInfo(node) - require.EqualValues(t, []netmapclient.AddPeerPrm{addPeerExp}, nc.addPeers, "invalid peers") - }) + require.EqualValues(t, []notaryInvoke{ + { + contract: nc.contractAddress, + fee: 0, + nonce: ev.NotaryRequest().MainTransaction.Nonce, + vub: nil, + method: "addPeerIR", + args: []any{node.Marshal()}, + }, + }, nc.notaryInvokes, "invalid notary invokes") } func TestUpdateState(t *testing.T) { t.Parallel() - t.Run("with notary", func(t *testing.T) { - t.Parallel() - ns := &testNodeStateSettings{ - maintAllowed: true, - } - nc := &testNetmapClient{} + ns := &testNodeStateSettings{ + maintAllowed: true, + } + nc := &testNetmapClient{} - proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true - p.NodeStateSettings = ns - p.NetmapClient = nc - }) - - require.NoError(t, err, "failed to create processor") - - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key") - - ev := netmapEvent.UpdatePeer{ - State: netmapContract.NodeStateOnline, - PubKey: key, - Request: &payload.P2PNotaryRequest{ - MainTransaction: &transaction.Transaction{ - Nonce: 100, - }, - }, - } - proc.handleUpdateState(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - require.EqualValues(t, []*transaction.Transaction{ - ev.Request.MainTransaction, - }, nc.invokedTxs, "invalid invoked transactions") + proc, err := newTestProc(t, func(p *Params) { + p.NetmapClient = nc + p.NodeStateSettings = ns }) - t.Run("without notary", func(t *testing.T) { - t.Parallel() - ns := &testNodeStateSettings{ - maintAllowed: true, - } - nc := &testNetmapClient{} + require.NoError(t, err, "failed to create processor") - proc, err := newTestProc(t, func(p *Params) { - p.NetmapClient = nc - p.NodeStateSettings = ns - }) + key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key") - require.NoError(t, err, "failed to create processor") + ev := netmapEvent.UpdatePeer{ + State: netmapContract.NodeStateOnline, + PubKey: key, + Request: &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + }, + } + proc.handleUpdateState(ev) - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key") + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } - ev := netmapEvent.UpdatePeer{ - State: netmapContract.NodeStateOnline, - PubKey: key, - } - proc.handleUpdateState(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - var expUpdPeer netmapclient.UpdatePeerPrm - expUpdPeer.SetMaintenance() - expUpdPeer.SetOnline() - expUpdPeer.SetKey(ev.PubKey.Bytes()) - - require.EqualValues(t, []netmapclient.UpdatePeerPrm{expUpdPeer}, nc.peerStateUpdates, "invalid peer state updates") - }) + require.EqualValues(t, []*transaction.Transaction{ev.Request.MainTransaction}, nc.invokedTxs, "invalid transactions") } func TestCleanupTick(t *testing.T) { t.Parallel() - t.Run("notary disabled", func(t *testing.T) { - t.Parallel() - - nc := &testNetmapClient{} - - proc, err := newTestProc(t, func(p *Params) { + nc := &testNetmapClient{ + contractAddress: util.Uint160{111}, + } + proc, err := newTestProc(t, + func(p *Params) { p.NetmapClient = nc - p.NotaryDisabled = true p.CleanupEnabled = true - }) + }, + ) - require.NoError(t, err, "failed to create processor") + require.NoError(t, err, "failed to create processor") - key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" - proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 95, - removeFlag: false, - }, - } - key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" - proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 98, - removeFlag: false, - }, - } + key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" + proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ + epochStamp: epochStamp{ + epoch: 95, + removeFlag: false, + }, + } + key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" + proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ + epochStamp: epochStamp{ + epoch: 98, + removeFlag: false, + }, + } - ev := netmapCleanupTick{ - epoch: 100, - txHash: util.Uint256{123}, - } + ev := netmapCleanupTick{ + epoch: 100, + txHash: util.Uint256{123}, + } - proc.handleCleanupTick(ev) + proc.handleCleanupTick(ev) - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } - keyExp, err := keys.NewPublicKeyFromString(key1Str) - require.NoError(t, err, "failed to parse expired key") + keyExp, err := keys.NewPublicKeyFromString(key1Str) + require.NoError(t, err, "failed to parse expired key") - updExp := netmapclient.UpdatePeerPrm{} - updExp.SetKey(keyExp.Bytes()) - updExp.SetHash(ev.TxHash()) + updExp := netmapclient.UpdatePeerPrm{} + updExp.SetKey(keyExp.Bytes()) + updExp.SetHash(ev.TxHash()) - require.EqualValues(t, []netmapclient.UpdatePeerPrm{updExp}, nc.peerStateUpdates, "invalid peer updates") - require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") - require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") - }) - - t.Run("notary enabled", func(t *testing.T) { - t.Parallel() - - nc := &testNetmapClient{ - contractAddress: util.Uint160{111}, - } - proc, err := newTestProc(t, - func(p *Params) { - p.NetmapClient = nc - p.CleanupEnabled = true - }, - ) - - require.NoError(t, err, "failed to create processor") - - key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" - proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 95, - removeFlag: false, - }, - } - key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" - proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 98, - removeFlag: false, - }, - } - - ev := netmapCleanupTick{ - epoch: 100, - txHash: util.Uint256{123}, - } - - proc.handleCleanupTick(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - keyExp, err := keys.NewPublicKeyFromString(key1Str) - require.NoError(t, err, "failed to parse expired key") - - updExp := netmapclient.UpdatePeerPrm{} - updExp.SetKey(keyExp.Bytes()) - updExp.SetHash(ev.TxHash()) - - require.EqualValues(t, []notaryInvoke{ - { - contract: nc.contractAddress, - fee: 0, - nonce: uint32(ev.epoch), - vub: nil, - method: "updateStateIR", - args: []any{int64(v2netmap.Offline), keyExp.Bytes()}, - }, - }, nc.notaryInvokes, "invalid notary invokes") - require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") - require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") - }) + require.EqualValues(t, []notaryInvoke{ + { + contract: nc.contractAddress, + fee: 0, + nonce: uint32(ev.epoch), + vub: nil, + method: "updateStateIR", + args: []any{int64(v2netmap.Offline), keyExp.Bytes()}, + }, + }, nc.notaryInvokes, "invalid notary invokes") + require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") + require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") } func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { @@ -407,7 +268,6 @@ func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { PoolSize: 1, CleanupEnabled: false, CleanupThreshold: 3, - NotaryDisabled: false, NodeStateSettings: ns, NodeValidator: &testValidator{}, EpochState: es, @@ -500,17 +360,11 @@ type testNetmapClient struct { netmap *netmap.NetMap txHeights map[util.Uint256]uint32 - peerStateUpdates []netmapclient.UpdatePeerPrm - notaryInvokes []notaryInvoke - newEpochs []uint64 - addPeers []netmapclient.AddPeerPrm - invokedTxs []*transaction.Transaction + notaryInvokes []notaryInvoke + newEpochs []uint64 + invokedTxs []*transaction.Transaction } -func (c *testNetmapClient) UpdatePeerState(p netmapclient.UpdatePeerPrm) error { - c.peerStateUpdates = append(c.peerStateUpdates, p) - return nil -} func (c *testNetmapClient) MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { c.notaryInvokes = append(c.notaryInvokes, notaryInvoke{ contract: contract, @@ -522,32 +376,35 @@ func (c *testNetmapClient) MorphNotaryInvoke(contract util.Uint160, fee fixedn.F }) return nil } + func (c *testNetmapClient) ContractAddress() util.Uint160 { return c.contractAddress } + func (c *testNetmapClient) EpochDuration() (uint64, error) { return c.epochDuration, nil } + func (c *testNetmapClient) MorphTxHeight(h util.Uint256) (uint32, error) { if res, found := c.txHeights[h]; found { return res, nil } return 0, fmt.Errorf("not found") } + func (c *testNetmapClient) NetMap() (*netmap.NetMap, error) { return c.netmap, nil } + func (c *testNetmapClient) NewEpoch(epoch uint64, force bool) error { c.newEpochs = append(c.newEpochs, epoch) return nil } + func (c *testNetmapClient) MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) { return true, nil } -func (c *testNetmapClient) AddPeer(p netmapclient.AddPeerPrm) error { - c.addPeers = append(c.addPeers, p) - return nil -} + func (c *testNetmapClient) MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error { c.invokedTxs = append(c.invokedTxs, mainTx) return nil diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index 45a08b377..170c39e2c 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -3,16 +3,15 @@ package netmap import ( v2netmap "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.uber.org/zap" ) -func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { +func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick) - return + return true } err := np.netmapSnapshot.forEachRemoveCandidate(ev.epoch, func(s string) error { @@ -31,23 +30,14 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { // See https://github.com/nspcc-dev/frostfs-contract/issues/225 const methodUpdateStateNotary = "updateStateIR" - if np.notaryDisabled { - prm := netmapclient.UpdatePeerPrm{} - - prm.SetKey(key.Bytes()) - prm.SetHash(ev.TxHash()) - - err = np.netmapClient.UpdatePeerState(prm) - } else { - err = np.netmapClient.MorphNotaryInvoke( - np.netmapClient.ContractAddress(), - 0, - uint32(ev.epoch), - nil, - methodUpdateStateNotary, - int64(v2netmap.Offline), key.Bytes(), - ) - } + err = np.netmapClient.MorphNotaryInvoke( + np.netmapClient.ContractAddress(), + 0, + uint32(ev.epoch), + nil, + methodUpdateStateNotary, + int64(v2netmap.Offline), key.Bytes(), + ) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapUpdateState, zap.Error(err)) } @@ -57,5 +47,8 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { if err != nil { np.log.Warn(logs.NetmapCantIterateOnNetmapCleanerCache, zap.String("error", err.Error())) + return false } + + return true } diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index b655db9aa..01bfbae67 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -10,7 +10,7 @@ import ( // Process new epoch notification by setting global epoch value and resetting // local epoch timer. -func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { +func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool { epoch := ev.EpochNumber() epochDuration, err := np.netmapClient.EpochDuration() @@ -41,7 +41,7 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { np.log.Warn(logs.NetmapCantGetNetmapSnapshotToPerformCleanup, zap.String("error", err.Error())) - return + return false } prm := cntClient.StartEstimationPrm{} @@ -63,13 +63,15 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { np.handleCleanupTick(netmapCleanupTick{epoch: epoch, txHash: ev.TxHash()}) np.handleAlphabetSync(governance.NewSyncEvent(ev.TxHash())) np.handleNotaryDeposit(ev) + + return true } // Process new epoch tick by invoking new epoch method in network map contract. -func (np *Processor) processNewEpochTick() { +func (np *Processor) processNewEpochTick() bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewEpochTick) - return + return true } nextEpoch := np.epochState.EpochCounter() + 1 @@ -78,5 +80,8 @@ func (np *Processor) processNewEpochTick() { err := np.netmapClient.NewEpoch(nextEpoch, false) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 9e6eeb53e..96b8c8e97 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -12,23 +12,21 @@ import ( // Process add peer notification by sanity check of new node // local epoch timer. -func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { +func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewPeerNotification) - return + return true } // check if notary transaction is valid, see #976 - if originalRequest := ev.NotaryRequest(); originalRequest != nil { - tx := originalRequest.MainTransaction - ok, err := np.netmapClient.MorphIsValidScript(tx.Script, tx.Signers) - if err != nil || !ok { - np.log.Warn(logs.NetmapNonhaltNotaryTransaction, - zap.String("method", "netmap.AddPeer"), - zap.String("hash", tx.Hash().StringLE()), - zap.Error(err)) - return - } + tx := ev.NotaryRequest().MainTransaction + ok, err := np.netmapClient.MorphIsValidScript(tx.Script, tx.Signers) + if err != nil || !ok { + np.log.Warn(logs.NetmapNonhaltNotaryTransaction, + zap.String("method", "netmap.AddPeer"), + zap.String("hash", tx.Hash().StringLE()), + zap.Error(err)) + return false } // unmarshal node info @@ -36,17 +34,17 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { if err := nodeInfo.Unmarshal(ev.Node()); err != nil { // it will be nice to have tx id at event structure to log it np.log.Warn(logs.NetmapCantParseNetworkMapCandidate) - return + return false } // validate and update node info - err := np.nodeValidator.VerifyAndUpdate(&nodeInfo) + err = np.nodeValidator.VerifyAndUpdate(&nodeInfo) if err != nil { np.log.Warn(logs.NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate, zap.String("error", err.Error()), ) - return + return false } // sort attributes to make it consistent @@ -71,32 +69,30 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { // See https://github.com/nspcc-dev/frostfs-contract/issues/154. const methodAddPeerNotary = "addPeerIR" - if nr := ev.NotaryRequest(); nr != nil { - // create new notary request with the original nonce - err = np.netmapClient.MorphNotaryInvoke( - np.netmapClient.ContractAddress(), - 0, - nr.MainTransaction.Nonce, - nil, - methodAddPeerNotary, - nodeInfoBinary, - ) - } else { - // notification event case - err = np.netmapClient.AddPeer(prm) - } + // create new notary request with the original nonce + err = np.netmapClient.MorphNotaryInvoke( + np.netmapClient.ContractAddress(), + 0, + ev.NotaryRequest().MainTransaction.Nonce, + nil, + methodAddPeerNotary, + nodeInfoBinary, + ) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) + return false } } + + return true } // Process update peer notification by sending approval tx to the smart contract. -func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { +func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreUpdatePeerNotification) - return + return true } // flag node to remove from local view, so it can be re-bootstrapped @@ -112,27 +108,14 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { zap.Error(err), ) - return + return false } } - if nr := ev.NotaryRequest(); nr != nil { - err = np.netmapClient.MorphNotarySignAndInvokeTX(nr.MainTransaction) - } else { - prm := netmapclient.UpdatePeerPrm{} - - switch { - case ev.Online(): - prm.SetOnline() - case ev.Maintenance(): - prm.SetMaintenance() - } - - prm.SetKey(ev.PublicKey().Bytes()) - - err = np.netmapClient.UpdatePeerState(prm) - } - if err != nil { + if err = np.netmapClient.MorphNotarySignAndInvokeTX(ev.NotaryRequest().MainTransaction); err != nil { np.log.Error(logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 522fa8b86..6b8a24a62 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -5,9 +5,9 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -55,7 +55,6 @@ type ( } Client interface { - UpdatePeerState(p netmapclient.UpdatePeerPrm) error MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error ContractAddress() util.Uint160 EpochDuration() (uint64, error) @@ -63,7 +62,6 @@ type ( NetMap() (*netmap.NetMap, error) NewEpoch(epoch uint64, force bool) error MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) - AddPeer(p netmapclient.AddPeerPrm) error MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error } @@ -75,6 +73,7 @@ type ( // and new epoch ticker, because it is related to contract. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool epochTimer EpochTimerReseter epochState EpochState @@ -90,14 +89,13 @@ type ( nodeValidator NodeValidator - notaryDisabled bool - nodeStateSettings state.NetworkSettings } // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int NetmapClient Client EpochTimer EpochTimerReseter @@ -112,16 +110,12 @@ type ( NodeValidator NodeValidator - NotaryDisabled bool - NodeStateSettings state.NetworkSettings } ) const ( - newEpochNotification = "NewEpoch" - addPeerNotification = "AddPeer" - updatePeerStateNotification = "UpdateState" + newEpochNotification = "NewEpoch" ) // New creates network map contract processor instance. @@ -154,8 +148,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/netmap: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, epochTimer: p.EpochTimer, epochState: p.EpochState, @@ -170,8 +170,6 @@ func New(p *Params) (*Processor, error) { nodeValidator: p.NodeValidator, - notaryDisabled: p.NotaryDisabled, - nodeStateSettings: p.NodeStateSettings, }, nil } @@ -189,20 +187,6 @@ func (np *Processor) ListenerNotificationParsers() []event.NotificationParserInf p.SetParser(netmapEvent.ParseNewEpoch) parsers = append(parsers, p) - if !np.notaryDisabled { - return parsers - } - - // new peer event - p.SetType(addPeerNotification) - p.SetParser(netmapEvent.ParseAddPeer) - parsers = append(parsers, p) - - // update peer event - p.SetType(updatePeerStateNotification) - p.SetParser(netmapEvent.ParseUpdatePeer) - parsers = append(parsers, p) - return parsers } @@ -219,20 +203,6 @@ func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerI i.SetHandler(np.handleNewEpoch) handlers = append(handlers, i) - if !np.notaryDisabled { - return handlers - } - - // new peer handler - i.SetType(addPeerNotification) - i.SetHandler(np.handleAddPeer) - handlers = append(handlers, i) - - // update peer handler - i.SetType(updatePeerStateNotification) - i.SetHandler(np.handleUpdateState) - handlers = append(handlers, i) - return handlers } diff --git a/pkg/innerring/processors/util.go b/pkg/innerring/processors/util.go new file mode 100644 index 000000000..364ffe25e --- /dev/null +++ b/pkg/innerring/processors/util.go @@ -0,0 +1,16 @@ +package processors + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" + "github.com/panjf2000/ants/v2" +) + +func SubmitEvent(pool *ants.Pool, metrics metrics.Register, eventLabel string, eventProcessor func() bool) error { + return pool.Submit(func() { + start := time.Now() + success := eventProcessor() + metrics.AddEvent(time.Since(start), eventLabel, success) + }) +} diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 6a6ca0ade..c5adb71eb 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -29,8 +29,8 @@ func (s *Server) EpochCounter() uint64 { // epoch counter. func (s *Server) SetEpochCounter(val uint64) { s.epochCounter.Store(val) - if s.metrics != nil { - s.metrics.SetEpoch(val) + if s.irMetrics != nil { + s.irMetrics.SetEpoch(val) } } @@ -153,15 +153,15 @@ func (s *Server) ResetEpochTimer(h uint32) error { } func (s *Server) setHealthStatus(hs control.HealthStatus) { - s.healthStatus.Store(hs) - if s.metrics != nil { - s.metrics.SetHealth(int32(hs)) + s.healthStatus.Store(int32(hs)) + if s.irMetrics != nil { + s.irMetrics.SetHealth(int32(hs)) } } // HealthStatus returns the current health status of the IR application. func (s *Server) HealthStatus() control.HealthStatus { - return s.healthStatus.Load().(control.HealthStatus) + return control.HealthStatus(s.healthStatus.Load()) } func initPersistentStateStorage(cfg *viper.Viper) (*state.PersistentStorage, error) { diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index a49324406..d5741fba7 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -3,11 +3,11 @@ package blobovnicza import ( "io/fs" "os" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -31,6 +31,8 @@ type cfg struct { objSizeLimit uint64 log *logger.Logger + + metrics Metrics } type boltDBCfg struct { @@ -52,6 +54,7 @@ func defaultCfg(c *cfg) { fullSizeLimit: 1 << 30, // 1GB objSizeLimit: 1 << 20, // 1MB log: &logger.Logger{Logger: zap.L()}, + metrics: &NoopMetrics{}, } } @@ -112,3 +115,10 @@ func WithReadOnly(ro bool) Option { c.boltOptions.ReadOnly = ro } } + +// WithMetrics returns an option to set metrics storage. +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index 5deaf5e4a..05f8906ac 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -21,7 +21,7 @@ func testPutGet(t *testing.T, blz *Blobovnicza, addr oid.Address, sz uint64, ass var pPut PutPrm pPut.SetAddress(addr) pPut.SetMarshaledObject(data) - _, err := blz.Put(pPut) + _, err := blz.Put(context.Background(), pPut) if assertErrPut != nil { require.True(t, assertErrPut(err)) } else { diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 84274528a..da4c870bd 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -35,6 +35,9 @@ func (b *Blobovnicza) Open() error { ) b.boltDB, err = bbolt.Open(b.path, b.perm, b.boltOptions) + if err == nil { + b.metrics.IncOpenBlobovnizcaCount() + } return err } @@ -81,7 +84,9 @@ func (b *Blobovnicza) Init() error { return fmt.Errorf("can't determine DB size: %w", err) } - b.filled.Store(uint64(info.Size())) + sz := uint64(info.Size()) + b.filled.Store(sz) + b.metrics.AddSize(sz) return err } @@ -91,5 +96,10 @@ func (b *Blobovnicza) Close() error { zap.String("path", b.path), ) - return b.boltDB.Close() + err := b.boltDB.Close() + if err == nil { + b.metrics.DecOpenBlobovnizcaCount() + b.metrics.SubSize(b.filled.Load()) + } + return err } diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 29a587cc9..8fbd363db 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -3,8 +3,8 @@ package blobovnicza import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -38,13 +38,14 @@ func (p *DeletePrm) SetAddress(addr oid.Address) { func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Delete", trace.WithAttributes( + attribute.String("path", b.path), attribute.String("address", prm.addr.EncodeToString()), )) defer span.End() addrKey := addressKey(prm.addr) - removed := false + found := false err := b.boltDB.Update(func(tx *bbolt.Tx) error { return b.iterateBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { @@ -56,9 +57,6 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err sz := uint64(len(objData)) - // decrease fullness counter - b.decSize(sz) - // remove object from the bucket err := buck.Delete(addrKey) @@ -67,16 +65,18 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err zap.String("binary size", stringifyByteSize(sz)), zap.String("range", stringifyBounds(lower, upper)), ) + // decrease fullness counter + b.decSize(sz) } - removed = true + found = true // stop iteration return true, err }) }) - if err == nil && !removed { + if err == nil && !found { var errNotFound apistatus.ObjectNotFound return DeleteRes{}, errNotFound diff --git a/pkg/local_object_storage/blobovnicza/exists.go b/pkg/local_object_storage/blobovnicza/exists.go index 8ac45c4aa..e6d28f938 100644 --- a/pkg/local_object_storage/blobovnicza/exists.go +++ b/pkg/local_object_storage/blobovnicza/exists.go @@ -1,17 +1,30 @@ package blobovnicza import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Exists check if object with the specified address is stored in b. -func (b *Blobovnicza) Exists(addr oid.Address) (bool, error) { +func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error) { var ( - exists bool - addrKey = addressKey(addr) + exists = false ) + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Exists", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + + addrKey := addressKey(addr) + err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(_ []byte, buck *bbolt.Bucket) error { exists = buck.Get(addrKey) != nil diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index c1cd19e53..d492b7559 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -46,6 +46,7 @@ var errInterruptForEach = errors.New("interrupt for-each") func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Get", trace.WithAttributes( + attribute.String("path", b.path), attribute.String("address", prm.addr.EncodeToString()), )) defer span.End() diff --git a/pkg/local_object_storage/blobovnicza/get_test.go b/pkg/local_object_storage/blobovnicza/get_test.go index ad30e8d94..40c434eb7 100644 --- a/pkg/local_object_storage/blobovnicza/get_test.go +++ b/pkg/local_object_storage/blobovnicza/get_test.go @@ -41,7 +41,7 @@ func TestBlobovnicza_Get(t *testing.T) { addr := oidtest.Address() obj := make([]byte, firstBucketBound+1) - exists, err := blz.Exists(addr) + exists, err := blz.Exists(context.Background(), addr) require.NoError(t, err) require.False(t, exists) @@ -50,7 +50,7 @@ func TestBlobovnicza_Get(t *testing.T) { prmPut.SetMarshaledObject(obj) // place object to [32K:64K] bucket - _, err = blz.Put(prmPut) + _, err = blz.Put(context.Background(), prmPut) require.NoError(t, err) var prmGet GetPrm @@ -61,7 +61,7 @@ func TestBlobovnicza_Get(t *testing.T) { require.NoError(t, err) require.Equal(t, obj, res.Object()) - exists, err := blz.Exists(addr) + exists, err := blz.Exists(context.Background(), addr) require.NoError(t, err) require.True(t, exists) } diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index 1adfacbc0..c2031ea54 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -1,10 +1,14 @@ package blobovnicza import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) func (b *Blobovnicza) iterateBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error { @@ -117,12 +121,26 @@ type IterateRes struct { // Returns handler's errors directly. Returns nil after iterating finish. // // Handler should not retain object data. Handler must not be nil. -func (b *Blobovnicza) Iterate(prm IteratePrm) (IterateRes, error) { +func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Iterate", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.Bool("decode_addresses", prm.decodeAddresses), + attribute.Bool("without_data", prm.withoutData), + attribute.Bool("ignore_errors", prm.ignoreErrors), + )) + defer span.End() + var elem IterationElement if err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(name []byte, buck *bbolt.Bucket) error { return buck.ForEach(func(k, v []byte) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } if prm.decodeAddresses { if err := addressFromKey(&elem.addr, k); err != nil { if prm.ignoreErrors { @@ -147,7 +165,7 @@ func (b *Blobovnicza) Iterate(prm IteratePrm) (IterateRes, error) { } // IterateAddresses is a helper function which iterates over Blobovnicza and passes addresses of the objects to f. -func IterateAddresses(blz *Blobovnicza, f func(oid.Address) error) error { +func IterateAddresses(ctx context.Context, blz *Blobovnicza, f func(oid.Address) error) error { var prm IteratePrm prm.DecodeAddresses() @@ -157,7 +175,7 @@ func IterateAddresses(blz *Blobovnicza, f func(oid.Address) error) error { return f(elem.Address()) }) - _, err := blz.Iterate(prm) + _, err := blz.Iterate(ctx, prm) return err } diff --git a/pkg/local_object_storage/blobovnicza/iterate_test.go b/pkg/local_object_storage/blobovnicza/iterate_test.go index 6ecb20c77..90308723c 100644 --- a/pkg/local_object_storage/blobovnicza/iterate_test.go +++ b/pkg/local_object_storage/blobovnicza/iterate_test.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "errors" "path/filepath" "testing" @@ -19,7 +20,7 @@ func TestBlobovniczaIterate(t *testing.T) { data := [][]byte{{0, 1, 2, 3}, {5, 6, 7, 8}} addr := oidtest.Address() - _, err := b.Put(PutPrm{addr: addr, objData: data[0]}) + _, err := b.Put(context.Background(), PutPrm{addr: addr, objData: data[0]}) require.NoError(t, err) require.NoError(t, b.boltDB.Update(func(tx *bbolt.Tx) error { @@ -33,22 +34,22 @@ func TestBlobovniczaIterate(t *testing.T) { return nil } - _, err = b.Iterate(IteratePrm{handler: inc}) + _, err = b.Iterate(context.Background(), IteratePrm{handler: inc}) require.NoError(t, err) require.ElementsMatch(t, seen, data) seen = seen[:0] - _, err = b.Iterate(IteratePrm{handler: inc, decodeAddresses: true}) + _, err = b.Iterate(context.Background(), IteratePrm{handler: inc, decodeAddresses: true}) require.Error(t, err) seen = seen[:0] - _, err = b.Iterate(IteratePrm{handler: inc, decodeAddresses: true, ignoreErrors: true}) + _, err = b.Iterate(context.Background(), IteratePrm{handler: inc, decodeAddresses: true, ignoreErrors: true}) require.NoError(t, err) require.ElementsMatch(t, seen, data[:1]) seen = seen[:0] expectedErr := errors.New("stop iteration") - _, err = b.Iterate(IteratePrm{ + _, err = b.Iterate(context.Background(), IteratePrm{ decodeAddresses: true, handler: func(IterationElement) error { return expectedErr }, ignoreErrors: true, diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go new file mode 100644 index 000000000..d511f90f1 --- /dev/null +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -0,0 +1,16 @@ +package blobovnicza + +type Metrics interface { + IncOpenBlobovnizcaCount() + DecOpenBlobovnizcaCount() + + AddSize(size uint64) + SubSize(size uint64) +} + +type NoopMetrics struct{} + +func (m *NoopMetrics) IncOpenBlobovnizcaCount() {} +func (m *NoopMetrics) DecOpenBlobovnizcaCount() {} +func (m *NoopMetrics) AddSize(uint64) {} +func (m *NoopMetrics) SubSize(uint64) {} diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index 37ed57e1c..cc816b3e1 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -1,11 +1,15 @@ package blobovnicza import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // PutPrm groups the parameters of Put operation. @@ -47,7 +51,15 @@ func (p *PutPrm) SetMarshaledObject(data []byte) { // Returns ErrFull if blobovnicza is filled. // // Should not be called in read-only configuration. -func (b *Blobovnicza) Put(prm PutPrm) (PutRes, error) { +func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Put", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.String("address", prm.addr.EncodeToString()), + attribute.Int("size", len(prm.objData)), + )) + defer span.End() + sz := uint64(len(prm.objData)) bucketName := bucketForSize(sz) key := addressKey(prm.addr) diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index 82454fa28..7e10b728e 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -41,10 +41,12 @@ func upperPowerOfTwo(v uint64) uint64 { func (b *Blobovnicza) incSize(sz uint64) { b.filled.Add(sz) + b.metrics.AddSize(sz) } func (b *Blobovnicza) decSize(sz uint64) { - b.filled.Sub(sz) + b.filled.Add(^(sz - 1)) + b.metrics.SubSize(sz) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index af976f977..1b0af342d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -216,7 +216,7 @@ func addressHash(addr *oid.Address, path string) uint64 { a = addr.EncodeToString() } - return hrw.Hash([]byte(a + path)) + return hrw.StringHash(a + path) } // converts uint64 to hex string. @@ -256,3 +256,7 @@ func (b *Blobovniczas) SetCompressor(cc *compression.Config) { func (b *Blobovniczas) SetReportErrorFunc(f func(string, error)) { b.reportError = f } + +func (b *Blobovniczas) SetParentID(parentID string) { + b.metrics.SetParentID(parentID) +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 0240c7a97..bc3d7d60c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "fmt" "path/filepath" @@ -12,6 +13,7 @@ import ( // Open opens blobovnicza tree. func (b *Blobovniczas) Open(readOnly bool) error { b.readOnly = readOnly + b.metrics.SetMode(readOnly) return nil } @@ -26,7 +28,7 @@ func (b *Blobovniczas) Init() error { return nil } - return b.iterateLeaves(func(p string) (bool, error) { + return b.iterateLeaves(context.TODO(), func(p string) (bool, error) { blz, err := b.openBlobovniczaNoCache(p) if err != nil { return true, err @@ -69,6 +71,7 @@ func (b *Blobovniczas) Close() error { } b.active = make(map[string]blobovniczaWithIndex) + b.metrics.Close() b.lruMtx.Unlock() @@ -122,9 +125,12 @@ func (b *Blobovniczas) openBlobovniczaNoCache(p string) (*blobovnicza.Blobovnicz b.openMtx.Lock() defer b.openMtx.Unlock() + path := filepath.Join(b.rootPath, p) + blz := blobovnicza.New(append(b.blzOpts, blobovnicza.WithReadOnly(b.readOnly), - blobovnicza.WithPath(filepath.Join(b.rootPath, p)), + blobovnicza.WithPath(path), + blobovnicza.WithMetrics(b.metrics.Blobovnizca()), )...) if err := blz.Open(); err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index f84d8fbe8..4d0801ef1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -4,12 +4,13 @@ import ( "context" "encoding/hex" "path/filepath" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -21,8 +22,17 @@ import ( // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res common.DeleteRes, err error) { + var ( + success = false + startedAt = time.Now() + ) + defer func() { + b.metrics.Delete(time.Since(startedAt), success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Delete", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), )) @@ -42,13 +52,16 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co return res, err } - return b.deleteObject(ctx, blz, bPrm) + if res, err = b.deleteObject(ctx, blz, bPrm); err == nil { + success = true + } + return res, err } activeCache := make(map[string]struct{}) objectFound := false - err = b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) // don't process active blobovnicza of the level twice @@ -78,6 +91,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co // not found in any blobovnicza return common.DeleteRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } + success = err == nil return } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index 9d9fd4cba..cd553ec30 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -4,11 +4,12 @@ import ( "context" "encoding/hex" "path/filepath" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -16,8 +17,18 @@ import ( // Exists implements common.Storage. func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + var ( + startedAt = time.Now() + success = false + found = false + ) + defer func() { + b.metrics.Exists(time.Since(startedAt), success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Exists", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), )) @@ -30,7 +41,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common return common.ExistsRes{}, err } - exists, err := blz.Exists(prm.Address) + exists, err := blz.Exists(ctx, prm.Address) return common.ExistsRes{Exists: exists}, err } @@ -39,8 +50,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common var gPrm blobovnicza.GetPrm gPrm.SetAddress(prm.Address) - var found bool - err := b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err := b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) _, ok := activeCache[dirPath] @@ -59,5 +69,6 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common return found, nil }) + success = err == nil return common.ExistsRes{Exists: found}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 0b8ccb64f..e88310f8a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -5,12 +5,13 @@ import ( "encoding/hex" "fmt" "path/filepath" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" @@ -23,8 +24,18 @@ import ( // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + b.metrics.Get(time.Since(startedAt), size, success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Get", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), attribute.Bool("raw", prm.Raw), @@ -41,12 +52,17 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G return res, err } - return b.getObject(ctx, blz, bPrm) + res, err = b.getObject(ctx, blz, bPrm) + if err == nil { + success = true + size = len(res.RawData) + } + return res, err } activeCache := make(map[string]struct{}) - err = b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) _, ok := activeCache[dirPath] @@ -72,6 +88,9 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G return res, logicerr.Wrap(apistatus.ObjectNotFound{}) } + success = true + size = len(res.RawData) + return } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index d6dfe51bd..8579502c0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -6,12 +6,13 @@ import ( "fmt" "path/filepath" "strconv" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" @@ -24,8 +25,18 @@ import ( // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + b.metrics.GetRange(time.Since(startedAt), size, success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.GetRange", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), @@ -40,13 +51,18 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re return common.GetRangeRes{}, err } - return b.getObjectRange(ctx, blz, prm) + res, err := b.getObjectRange(ctx, blz, prm) + if err == nil { + size = len(res.Data) + success = true + } + return res, err } activeCache := make(map[string]struct{}) objectFound := false - err = b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) _, ok := activeCache[dirPath] @@ -78,6 +94,11 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } + if err == nil { + success = true + size = len(res.Data) + } + return } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 9918801b9..0154fe2ca 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -1,18 +1,38 @@ package blobovniczatree import ( + "context" "fmt" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Iterate iterates over all objects in b. -func (b *Blobovniczas) Iterate(prm common.IteratePrm) (common.IterateRes, error) { - return common.IterateRes{}, b.iterateBlobovniczas(prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { +func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + var ( + startedAt = time.Now() + err error + ) + defer func() { + b.metrics.Iterate(time.Since(startedAt), err == nil) + }() + + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Iterate", + trace.WithAttributes( + attribute.String("path", b.rootPath), + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + + err = b.iterateBlobovniczas(ctx, prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { var subPrm blobovnicza.IteratePrm subPrm.SetHandler(func(elem blobovnicza.IterationElement) error { data, err := b.compression.Decompress(elem.ObjectData()) @@ -39,14 +59,15 @@ func (b *Blobovniczas) Iterate(prm common.IteratePrm) (common.IterateRes, error) }) subPrm.DecodeAddresses() - _, err := blz.Iterate(subPrm) + _, err := blz.Iterate(ctx, subPrm) return err }) + return common.IterateRes{}, err } // iterator over all Blobovniczas in unsorted order. Break on f's error return. -func (b *Blobovniczas) iterateBlobovniczas(ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error { - return b.iterateLeaves(func(p string) (bool, error) { +func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error { + return b.iterateLeaves(ctx, func(p string) (bool, error) { blz, err := b.openBlobovnicza(p) if err != nil { if ignoreErrors { @@ -62,8 +83,9 @@ func (b *Blobovniczas) iterateBlobovniczas(ignoreErrors bool, f func(string, *bl } // iterator over the paths of Blobovniczas sorted by weight. -func (b *Blobovniczas) iterateSortedLeaves(addr *oid.Address, f func(string) (bool, error)) error { +func (b *Blobovniczas) iterateSortedLeaves(ctx context.Context, addr *oid.Address, f func(string) (bool, error)) error { _, err := b.iterateSorted( + ctx, addr, make([]string, 0, b.blzShallowDepth), b.blzShallowDepth, @@ -74,13 +96,14 @@ func (b *Blobovniczas) iterateSortedLeaves(addr *oid.Address, f func(string) (bo } // iterator over directories with Blobovniczas sorted by weight. -func (b *Blobovniczas) iterateDeepest(addr oid.Address, f func(string) (bool, error)) error { +func (b *Blobovniczas) iterateDeepest(ctx context.Context, addr oid.Address, f func(string) (bool, error)) error { depth := b.blzShallowDepth if depth > 0 { depth-- } _, err := b.iterateSorted( + ctx, &addr, make([]string, 0, depth), depth, @@ -91,7 +114,7 @@ func (b *Blobovniczas) iterateDeepest(addr oid.Address, f func(string) (bool, er } // iterator over particular level of directories. -func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) { +func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) { indices := indexSlice(b.blzShallowWidth) hrw.SortSliceByValue(indices, addressHash(addr, filepath.Join(curPath...))) @@ -99,6 +122,11 @@ func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDe exec := uint64(len(curPath)) == execDepth for i := range indices { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } if i == 0 { curPath = append(curPath, u64ToHexString(indices[i])) } else { @@ -111,7 +139,7 @@ func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDe } else if stop { return true, nil } - } else if stop, err := b.iterateSorted(addr, curPath, execDepth, f); err != nil { + } else if stop, err := b.iterateSorted(ctx, addr, curPath, execDepth, f); err != nil { return false, err } else if stop { return true, nil @@ -122,8 +150,8 @@ func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDe } // iterator over the paths of Blobovniczas in random order. -func (b *Blobovniczas) iterateLeaves(f func(string) (bool, error)) error { - return b.iterateSortedLeaves(nil, f) +func (b *Blobovniczas) iterateLeaves(ctx context.Context, f func(string) (bool, error)) error { + return b.iterateSortedLeaves(ctx, nil, f) } // makes slice of uint64 values from 0 to number-1. diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go new file mode 100644 index 000000000..6c9b600c0 --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -0,0 +1,38 @@ +package blobovniczatree + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" +) + +type Metrics interface { + Blobovnizca() blobovnicza.Metrics + + SetParentID(parentID string) + + SetMode(readOnly bool) + Close() + + Delete(d time.Duration, success, withStorageID bool) + Exists(d time.Duration, success, withStorageID bool) + GetRange(d time.Duration, size int, success, withStorageID bool) + Get(d time.Duration, size int, success, withStorageID bool) + Iterate(d time.Duration, success bool) + Put(d time.Duration, size int, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetParentID(string) {} +func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) Delete(time.Duration, bool, bool) {} +func (m *noopMetrics) Exists(time.Duration, bool, bool) {} +func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Iterate(time.Duration, bool) {} +func (m *noopMetrics) Put(time.Duration, int, bool) {} +func (m *noopMetrics) Blobovnizca() blobovnicza.Metrics { + return &blobovnicza.NoopMetrics{} +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 95ef8635a..d0503f23b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -21,6 +21,7 @@ type cfg struct { blzOpts []blobovnicza.Option // reportError is the function called when encountering disk errors. reportError func(string, error) + metrics Metrics } type Option func(*cfg) @@ -40,6 +41,7 @@ func initConfig(c *cfg) { blzShallowDepth: defaultBlzShallowDepth, blzShallowWidth: defaultBlzShallowWidth, reportError: func(string, error) {}, + metrics: &noopMetrics{}, } } @@ -91,3 +93,9 @@ func WithObjectSizeLimit(sz uint64) Option { c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz)) } } + +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index ec302d143..038d5244d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -4,11 +4,12 @@ import ( "context" "errors" "path/filepath" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -19,6 +20,15 @@ import ( // // returns error if could not save object in any blobovnicza. func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + var ( + success bool + size int + startedAt = time.Now() + ) + defer func() { + b.metrics.Put(time.Since(startedAt), size, success) + }() + _, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Put", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -33,6 +43,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe if !prm.DontCompress { prm.RawData = b.compression.Compress(prm.RawData) } + size = len(prm.RawData) var putPrm blobovnicza.PutPrm putPrm.SetAddress(prm.Address) @@ -45,7 +56,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe PutPrm: putPrm, } - if err := b.iterateDeepest(prm.Address, it.iterate); err != nil { + if err := b.iterateDeepest(ctx, prm.Address, func(s string) (bool, error) { return it.iterate(ctx, s) }); err != nil { return common.PutRes{}, err } else if it.ID == nil { if it.AllFull { @@ -54,6 +65,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe return common.PutRes{}, errPutFailed } + success = true return common.PutRes{StorageID: it.ID.Bytes()}, nil } @@ -64,7 +76,7 @@ type putIterator struct { PutPrm blobovnicza.PutPrm } -func (i *putIterator) iterate(path string) (bool, error) { +func (i *putIterator) iterate(ctx context.Context, path string) (bool, error) { active, err := i.B.getActivated(path) if err != nil { if !isLogical(err) { @@ -77,7 +89,7 @@ func (i *putIterator) iterate(path string) (bool, error) { return false, nil } - if _, err := active.blz.Put(i.PutPrm); err != nil { + if _, err := active.blz.Put(ctx, i.PutPrm); err != nil { // Check if blobovnicza is full. We could either receive `blobovnicza.ErrFull` error // or update active blobovnicza in other thread. In the latter case the database will be closed // and `updateActive` takes care of not updating the active blobovnicza twice. @@ -99,7 +111,7 @@ func (i *putIterator) iterate(path string) (bool, error) { return false, nil } - return i.iterate(path) + return i.iterate(ctx, path) } i.AllFull = false diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index a6fe9935e..6c6aed87f 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -43,10 +43,12 @@ type cfg struct { compression compression.Config log *logger.Logger storage []SubStorage + metrics Metrics } func initConfig(c *cfg) { c.log = &logger.Logger{Logger: zap.L()} + c.metrics = &noopMetrics{} } // New creates, initializes and returns new BlobStor instance. @@ -70,6 +72,13 @@ func (b *BlobStor) SetLogger(l *logger.Logger) { b.log = l } +func (b *BlobStor) SetParentID(parentID string) { + b.metrics.SetParentID(parentID) + for _, ss := range b.storage { + ss.Storage.SetParentID(parentID) + } +} + // WithStorages provides sub-blobstors. func WithStorages(st []SubStorage) Option { return func(c *cfg) { @@ -113,3 +122,9 @@ func (b *BlobStor) SetReportErrorFunc(f func(string, error)) { b.storage[i].Storage.SetReportErrorFunc(f) } } + +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 801d32c1e..b808480e2 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -19,11 +19,12 @@ type Storage interface { // SetReportErrorFunc allows to provide a function to be called on disk errors. // This function MUST be called before Open. SetReportErrorFunc(f func(string, error)) + SetParentID(parentID string) Get(context.Context, GetPrm) (GetRes, error) GetRange(context.Context, GetRangePrm) (GetRangeRes, error) Exists(context.Context, ExistsPrm) (ExistsRes, error) Put(context.Context, PutPrm) (PutRes, error) Delete(context.Context, DeletePrm) (DeleteRes, error) - Iterate(IteratePrm) (IterateRes, error) + Iterate(context.Context, IteratePrm) (IterateRes, error) } diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index abe39575b..6b439dcf0 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -18,6 +18,7 @@ func (b *BlobStor) Open(readOnly bool) error { return err } } + b.metrics.SetMode(readOnly) return nil } @@ -65,5 +66,8 @@ func (b *BlobStor) Close() error { if firstErr == nil { firstErr = err } + if firstErr == nil { + b.metrics.Close() + } return firstErr } diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index 377214fb8..fe49fc46a 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -4,15 +4,24 @@ import ( "context" "encoding/hex" "errors" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + b.metrics.Delete(time.Since(startedAt), success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Delete", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -28,6 +37,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del res, err := b.storage[i].Storage.Delete(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { if err == nil { + success = true logOp(b.log, deleteOp, prm.Address, b.storage[i].Storage.Type(), prm.StorageID) } return res, err @@ -45,6 +55,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del res, err := st.Delete(ctx, prm) if err == nil { + success = true logOp(b.log, deleteOp, prm.Address, st.Type(), prm.StorageID) } diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 3c76764a9..03dad392a 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -3,10 +3,11 @@ package blobstor import ( "context" "encoding/hex" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -17,6 +18,14 @@ import ( // Returns any error encountered that did not allow // to completely check object existence. func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + var ( + exists = false + startedAt = time.Now() + ) + defer func() { + b.metrics.Exists(time.Since(startedAt), exists, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Exists", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -29,9 +38,13 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi if prm.StorageID != nil { if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.Exists(ctx, prm) + res, err := b.storage[len(b.storage)-1].Storage.Exists(ctx, prm) + exists = err == nil && res.Exists + return res, err } - return b.storage[0].Storage.Exists(ctx, prm) + res, err := b.storage[0].Storage.Exists(ctx, prm) + exists = err == nil && res.Exists + return res, err } // If there was an error during existence check below, @@ -47,6 +60,7 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi for i := range b.storage { res, err := b.storage[i].Storage.Exists(ctx, prm) if err == nil && res.Exists { + exists = true return res, nil } else if err != nil { errors = append(errors, err) diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index 1ff74893d..f41b7aacd 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -7,6 +7,7 @@ import ( // Open implements common.Storage. func (t *FSTree) Open(ro bool) error { t.readOnly = ro + t.metrics.SetMode(ro) return nil } @@ -16,4 +17,7 @@ func (t *FSTree) Init() error { } // Close implements common.Storage. -func (*FSTree) Close() error { return nil } +func (t *FSTree) Close() error { + t.metrics.Close() + return nil +} diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 75f63193a..9ca5d4bd9 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -11,12 +11,13 @@ import ( "strconv" "strings" "syscall" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -35,6 +36,7 @@ type FSTree struct { noSync bool readOnly bool + metrics Metrics } // Info groups the information about file storage. @@ -64,6 +66,7 @@ func New(opts ...Option) *FSTree { Config: nil, Depth: 4, DirNameLen: DirNameLen, + metrics: &noopMetrics{}, } for i := range opts { opts[i](f) @@ -100,11 +103,28 @@ func addressFromString(s string) (oid.Address, error) { } // Iterate iterates over all stored objects. -func (t *FSTree) Iterate(prm common.IteratePrm) (common.IterateRes, error) { - return common.IterateRes{}, t.iterate(0, []string{t.RootPath}, prm) +func (t *FSTree) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + var ( + err error + startedAt = time.Now() + ) + + defer func() { + t.metrics.Iterate(time.Since(startedAt), err == nil) + }() + + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Iterate", + trace.WithAttributes( + attribute.String("path", t.RootPath), + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + + err = t.iterate(ctx, 0, []string{t.RootPath}, prm) + return common.IterateRes{}, err } -func (t *FSTree) iterate(depth uint64, curPath []string, prm common.IteratePrm) error { +func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, prm common.IteratePrm) error { curName := strings.Join(curPath[1:], "") des, err := os.ReadDir(filepath.Join(curPath...)) if err != nil { @@ -119,10 +139,15 @@ func (t *FSTree) iterate(depth uint64, curPath []string, prm common.IteratePrm) curPath = append(curPath, "") for i := range des { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } curPath[l] = des[i].Name() if !isLast && des[i].IsDir() { - err := t.iterate(depth+1, curPath, prm) + err := t.iterate(ctx, depth+1, curPath, prm) if err != nil { // Must be error from handler in case errors are ignored. // Need to report. @@ -197,19 +222,29 @@ func (t *FSTree) treePath(addr oid.Address) string { // Delete removes the object with the specified address from the storage. func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + var ( + err error + startedAt = time.Now() + ) + defer func() { + t.metrics.Delete(time.Since(startedAt), err == nil) + }() + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Delete", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), )) defer span.End() if t.readOnly { - return common.DeleteRes{}, common.ErrReadOnly + err = common.ErrReadOnly + return common.DeleteRes{}, err } p := t.treePath(prm.Address) - err := os.Remove(p) + err = os.Remove(p) if err != nil && os.IsNotExist(err) { err = logicerr.Wrap(apistatus.ObjectNotFound{}) } @@ -219,8 +254,17 @@ func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Delet // Exists returns the path to the file with object contents if it exists in the storage // and an error otherwise. func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + var ( + success = false + startedAt = time.Now() + ) + defer func() { + t.metrics.Exists(time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Exists", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), )) defer span.End() @@ -232,27 +276,40 @@ func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exist if os.IsNotExist(err) { err = nil } + success = err == nil return common.ExistsRes{Exists: found}, err } // Put puts an object in the storage. func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + var ( + size int + startedAt = time.Now() + err error + ) + defer func() { + t.metrics.Put(time.Since(startedAt), size, err == nil) + }() + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Put", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.Bool("dont_compress", prm.DontCompress), )) defer span.End() if t.readOnly { - return common.PutRes{}, common.ErrReadOnly + err = common.ErrReadOnly + return common.PutRes{}, err } p := t.treePath(prm.Address) - if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { + if err = util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { if errors.Is(err, syscall.ENOSPC) { - return common.PutRes{}, common.ErrNoSpace + err = common.ErrNoSpace + return common.PutRes{}, err } return common.PutRes{}, err } @@ -282,17 +339,19 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err // to be so hecking simple. // In a very rare situation we can have multiple partially written copies on disk, // this will be fixed in another issue (we should remove garbage on start). + size = len(prm.RawData) const retryCount = 5 for i := 0; i < retryCount; i++ { tmpPath := p + "#" + strconv.FormatUint(uint64(i), 10) - err := t.writeAndRename(tmpPath, p, prm.RawData) + err = t.writeAndRename(tmpPath, p, prm.RawData) if err != syscall.EEXIST || i == retryCount-1 { return common.PutRes{StorageID: []byte{}}, err } } + err = fmt.Errorf("couldn't read file after %d retries", retryCount) // unreachable, but precaution never hurts, especially 1 day before release. - return common.PutRes{StorageID: []byte{}}, fmt.Errorf("couldn't read file after %d retries", retryCount) + return common.PutRes{StorageID: []byte{}}, err } // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. @@ -360,8 +419,18 @@ func (t *FSTree) PutStream(addr oid.Address, handler func(*os.File) error) error // Get returns an object from the storage by address. func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + t.metrics.Get(time.Since(startedAt), size, success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.Get", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.Bool("raw", prm.Raw), attribute.String("address", prm.Address.EncodeToString()), )) @@ -389,19 +458,30 @@ func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, err if err != nil { return common.GetRes{}, err } + size = len(data) obj := objectSDK.New() if err := obj.Unmarshal(data); err != nil { return common.GetRes{}, err } - - return common.GetRes{Object: obj, RawData: data}, err + success = true + return common.GetRes{Object: obj, RawData: data}, nil } // GetRange implements common.Storage. func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + t.metrics.GetRange(time.Since(startedAt), size, success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.GetRange", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), attribute.String("length", strconv.FormatUint(prm.Range.GetLength(), 10)), @@ -421,8 +501,11 @@ func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.G return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectOutOfRange{}) } + success = true + data := payload[from:to] + size = len(data) return common.GetRangeRes{ - Data: payload[from:to], + Data: data, }, nil } @@ -471,3 +554,7 @@ func (t *FSTree) SetCompressor(cc *compression.Config) { func (t *FSTree) SetReportErrorFunc(_ func(string, error)) { // Do nothing, FSTree can encounter only one error which is returned. } + +func (t *FSTree) SetParentID(parentID string) { + t.metrics.SetParentID(parentID) +} diff --git a/pkg/local_object_storage/blobstor/fstree/metrics.go b/pkg/local_object_storage/blobstor/fstree/metrics.go new file mode 100644 index 000000000..ca6a54975 --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/metrics.go @@ -0,0 +1,29 @@ +package fstree + +import "time" + +type Metrics interface { + SetParentID(parentID string) + + SetMode(readOnly bool) + Close() + + Iterate(d time.Duration, success bool) + Delete(d time.Duration, success bool) + Exists(d time.Duration, success bool) + Put(d time.Duration, size int, success bool) + Get(d time.Duration, size int, success bool) + GetRange(d time.Duration, size int, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetParentID(string) {} +func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) Iterate(time.Duration, bool) {} +func (m *noopMetrics) Delete(time.Duration, bool) {} +func (m *noopMetrics) Exists(time.Duration, bool) {} +func (m *noopMetrics) Put(time.Duration, int, bool) {} +func (m *noopMetrics) Get(time.Duration, int, bool) {} +func (m *noopMetrics) GetRange(time.Duration, int, bool) {} diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go index 07e547444..52c8718c2 100644 --- a/pkg/local_object_storage/blobstor/fstree/option.go +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -35,3 +35,9 @@ func WithNoSync(noSync bool) Option { f.noSync = noSync } } + +func WithMetrics(m Metrics) Option { + return func(f *FSTree) { + f.metrics = m + } +} diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index 65bc87c07..fab86ea5d 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -4,10 +4,11 @@ import ( "context" "encoding/hex" "errors" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -16,7 +17,14 @@ import ( // Get reads the object from b. // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. -func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { +func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { + var ( + startedAt = time.Now() + ) + defer func() { + b.metrics.Get(time.Since(startedAt), len(res.RawData), err == nil, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Get", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -30,7 +38,7 @@ func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, e if prm.StorageID == nil { for i := range b.storage { - res, err := b.storage[i].Storage.Get(ctx, prm) + res, err = b.storage[i].Storage.Get(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { return res, err } @@ -39,7 +47,9 @@ func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, e return common.GetRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.Get(ctx, prm) + res, err = b.storage[len(b.storage)-1].Storage.Get(ctx, prm) + } else { + res, err = b.storage[0].Storage.Get(ctx, prm) } - return b.storage[0].Storage.Get(ctx, prm) + return res, err } diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index ff9e72e97..671fbf02e 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -5,10 +5,11 @@ import ( "encoding/hex" "errors" "strconv" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -17,7 +18,14 @@ import ( // GetRange reads object payload data from b. // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. -func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { +func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { + var ( + startedAt = time.Now() + ) + defer func() { + b.metrics.GetRange(time.Since(startedAt), len(res.Data), err == nil, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.GetRange", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -32,7 +40,7 @@ func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (common if prm.StorageID == nil { for i := range b.storage { - res, err := b.storage[i].Storage.GetRange(ctx, prm) + res, err = b.storage[i].Storage.GetRange(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { return res, err } @@ -41,7 +49,9 @@ func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (common return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.GetRange(ctx, prm) + res, err = b.storage[len(b.storage)-1].Storage.GetRange(ctx, prm) + } else { + res, err = b.storage[0].Storage.GetRange(ctx, prm) } - return b.storage[0].Storage.GetRange(ctx, prm) + return res, err } diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index 83ada9607..34622c857 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -49,7 +49,7 @@ func runTestNormalHandler(t *testing.T, s common.Storage, objects []objectDesc) return nil } - _, err := s.Iterate(iterPrm) + _, err := s.Iterate(context.Background(), iterPrm) require.NoError(t, err) require.Equal(t, len(objects), len(seen)) for i := range objects { @@ -72,7 +72,7 @@ func runTestLazyHandler(t *testing.T, s common.Storage, objects []objectDesc) { return nil } - _, err := s.Iterate(iterPrm) + _, err := s.Iterate(context.Background(), iterPrm) require.NoError(t, err) require.Equal(t, len(objects), len(seen)) for i := range objects { @@ -107,7 +107,7 @@ func runTestIgnoreLogicalErrors(t *testing.T, s common.Storage, objects []object return nil } - _, err := s.Iterate(iterPrm) + _, err := s.Iterate(context.Background(), iterPrm) require.Equal(t, err, logicErr) require.Equal(t, len(objects)/2, len(seen)) for i := range objects { diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index 2c37ee776..5a41e4c4f 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -1,11 +1,16 @@ package blobstor import ( + "context" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -16,22 +21,36 @@ import ( // did not allow to completely iterate over the storage. // // If handler returns an error, method wraps and returns it immediately. -func (b *BlobStor) Iterate(prm common.IteratePrm) (common.IterateRes, error) { +func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + b.metrics.Iterate(time.Since(startedAt), success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Iterate", + trace.WithAttributes( + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + b.modeMtx.RLock() defer b.modeMtx.RUnlock() for i := range b.storage { - _, err := b.storage[i].Storage.Iterate(prm) + _, err := b.storage[i].Storage.Iterate(ctx, prm) if err != nil && !prm.IgnoreErrors { return common.IterateRes{}, fmt.Errorf("blobstor iterator failure: %w", err) } } + success = true return common.IterateRes{}, nil } // IterateBinaryObjects is a helper function which iterates over BlobStor and passes binary objects to f. // Errors related to object reading and unmarshaling are logged and skipped. -func IterateBinaryObjects(blz *BlobStor, f func(addr oid.Address, data []byte, descriptor []byte) error) error { +func IterateBinaryObjects(ctx context.Context, blz *BlobStor, f func(addr oid.Address, data []byte, descriptor []byte) error) error { var prm common.IteratePrm prm.Handler = func(elem common.IterationElement) error { @@ -45,7 +64,7 @@ func IterateBinaryObjects(blz *BlobStor, f func(addr oid.Address, data []byte, d return nil } - _, err := blz.Iterate(prm) + _, err := blz.Iterate(ctx, prm) return err } diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index 6488ff5fc..c35869655 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -68,7 +68,7 @@ func TestIterateObjects(t *testing.T) { require.NoError(t, err) } - err := IterateBinaryObjects(blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { + err := IterateBinaryObjects(context.Background(), blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { v, ok := mObjs[string(data)] require.True(t, ok) diff --git a/pkg/local_object_storage/blobstor/memstore/control.go b/pkg/local_object_storage/blobstor/memstore/control.go index 4deb9f6e2..e6943626b 100644 --- a/pkg/local_object_storage/blobstor/memstore/control.go +++ b/pkg/local_object_storage/blobstor/memstore/control.go @@ -13,3 +13,4 @@ func (s *memstoreImpl) Type() string { return Type } func (s *memstoreImpl) Path() string { return s.rootPath } func (s *memstoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc } func (s *memstoreImpl) SetReportErrorFunc(f func(string, error)) { s.reportError = f } +func (s *memstoreImpl) SetParentID(string) {} diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index e435cfef4..b6cca2551 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -126,7 +126,7 @@ func (s *memstoreImpl) Delete(_ context.Context, req common.DeletePrm) (common.D return common.DeleteRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } -func (s *memstoreImpl) Iterate(req common.IteratePrm) (common.IterateRes, error) { +func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common.IterateRes, error) { s.mu.RLock() defer s.mu.RUnlock() for k, v := range s.objs { diff --git a/pkg/local_object_storage/blobstor/metrics.go b/pkg/local_object_storage/blobstor/metrics.go new file mode 100644 index 000000000..4a7b40092 --- /dev/null +++ b/pkg/local_object_storage/blobstor/metrics.go @@ -0,0 +1,28 @@ +package blobstor + +import "time" + +type Metrics interface { + SetParentID(parentID string) + SetMode(readOnly bool) + Close() + + Delete(d time.Duration, success, withStorageID bool) + Exists(d time.Duration, success, withStorageID bool) + GetRange(d time.Duration, size int, success, withStorageID bool) + Get(d time.Duration, size int, success, withStorageID bool) + Iterate(d time.Duration, success bool) + Put(d time.Duration, size int, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetParentID(string) {} +func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) Delete(time.Duration, bool, bool) {} +func (m *noopMetrics) Exists(time.Duration, bool, bool) {} +func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Iterate(time.Duration, bool) {} +func (m *noopMetrics) Put(time.Duration, int, bool) {} diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index f21982530..5245146cb 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -207,7 +207,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) { // Benchmark iterate cnt := 0 b.ResetTimer() - _, err := st.Iterate(common.IteratePrm{ + _, err := st.Iterate(context.Background(), common.IteratePrm{ Handler: func(elem common.IterationElement) error { cnt++ return nil diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index 2ae7f0fe6..a748750dd 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -3,11 +3,12 @@ package blobstor import ( "context" "fmt" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -26,6 +27,15 @@ var ErrNoPlaceFound = logicerr.New("couldn't find a place to store an object") // Returns any error encountered that // did not allow to completely save the object. func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + b.metrics.Put(time.Since(startedAt), size, success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Put", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -47,11 +57,13 @@ func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e } prm.RawData = data } + size = len(prm.RawData) for i := range b.storage { if b.storage[i].Policy == nil || b.storage[i].Policy(prm.Object, prm.RawData) { res, err := b.storage[i].Storage.Put(ctx, prm) if err == nil { + success = true logOp(b.log, putOp, prm.Address, b.storage[i].Storage.Type(), res.StorageID) } return res, err diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 24d742fda..2508d74f6 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -202,15 +202,17 @@ func (s *TestStore) Delete(ctx context.Context, req common.DeletePrm) (common.De } } -func (s *TestStore) Iterate(req common.IteratePrm) (common.IterateRes, error) { +func (s *TestStore) Iterate(ctx context.Context, req common.IteratePrm) (common.IterateRes, error) { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.Iterate != nil: return s.overrides.Iterate(req) case s.st != nil: - return s.st.Iterate(req) + return s.st.Iterate(ctx, req) default: panic(fmt.Sprintf("unexpected storage call: Iterate(%+v)", req)) } } + +func (s *TestStore) SetParentID(string) {} diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 034837110..e45f502ac 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -1,6 +1,8 @@ package engine import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" @@ -67,7 +69,7 @@ func ContainerSize(e *StorageEngine, id cid.ID) (uint64, error) { func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRes, err error) { if e.metrics != nil { - defer elapsed(e.metrics.AddEstimateContainerSizeDuration)() + defer elapsed("EstimateContainerSize", e.metrics.AddMethodDuration)() } e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { @@ -92,9 +94,9 @@ func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRe // ListContainers returns a unique container IDs presented in the engine objects. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) ListContainers(_ ListContainersPrm) (res ListContainersRes, err error) { +func (e *StorageEngine) ListContainers(ctx context.Context, _ ListContainersPrm) (res ListContainersRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.listContainers() + res, err = e.listContainers(ctx) return err }) @@ -102,10 +104,10 @@ func (e *StorageEngine) ListContainers(_ ListContainersPrm) (res ListContainersR } // ListContainers calls ListContainers method on engine to get a unique container IDs presented in the engine objects. -func ListContainers(e *StorageEngine) ([]cid.ID, error) { +func ListContainers(ctx context.Context, e *StorageEngine) ([]cid.ID, error) { var prm ListContainersPrm - res, err := e.ListContainers(prm) + res, err := e.ListContainers(ctx, prm) if err != nil { return nil, err } @@ -113,15 +115,15 @@ func ListContainers(e *StorageEngine) ([]cid.ID, error) { return res.Containers(), nil } -func (e *StorageEngine) listContainers() (ListContainersRes, error) { +func (e *StorageEngine) listContainers(ctx context.Context) (ListContainersRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddListContainersDuration)() + defer elapsed("ListContainers", e.metrics.AddMethodDuration)() } uniqueIDs := make(map[string]cid.ID) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - res, err := sh.Shard.ListContainers(shard.ListContainersPrm{}) + res, err := sh.Shard.ListContainers(ctx, shard.ListContainersPrm{}) if err != nil { e.reportShardError(sh, "can't get list of containers", err) return false diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 9ad4fcf9c..bd166b3ff 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) type shardInitError struct { @@ -74,23 +75,30 @@ func (e *StorageEngine) Init(ctx context.Context) error { e.mtx.Lock() defer e.mtx.Unlock() - var wg sync.WaitGroup var errCh = make(chan shardInitError, len(e.shards)) + var eg errgroup.Group + if e.cfg.lowMem && e.anyShardRequiresRefill() { + eg.SetLimit(1) + } for id, sh := range e.shards { - wg.Add(1) - go func(id string, sh *shard.Shard) { - defer wg.Done() + id := id + sh := sh + eg.Go(func() error { if err := sh.Init(ctx); err != nil { errCh <- shardInitError{ err: err, id: id, } } - }(id, sh.Shard) + return nil + }) } - wg.Wait() + err := eg.Wait() close(errCh) + if err != nil { + return fmt.Errorf("failed to initialize shards: %w", err) + } for res := range errCh { if res.err != nil { @@ -125,6 +133,15 @@ func (e *StorageEngine) Init(ctx context.Context) error { return nil } +func (e *StorageEngine) anyShardRequiresRefill() bool { + for _, sh := range e.shards { + if sh.NeedRefillMetabase() { + return true + } + } + return false +} + var errClosed = errors.New("storage engine is closed") // Close releases all StorageEngine's components. Waits for all data-related operations to complete. diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index d7eaae1d8..0c433f226 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strconv" + "sync/atomic" "testing" "time" @@ -24,7 +25,6 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zaptest" ) diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index f9b9c9a87..4d6d838bc 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -4,9 +4,9 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -67,7 +67,7 @@ func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRe func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddDeleteDuration)() + defer elapsed("Delete", e.metrics.AddMethodDuration)() } var locked struct { diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 53c62981c..bbc27615a 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -18,6 +18,8 @@ import ( ) func TestDeleteBigObject(t *testing.T) { + t.Parallel() + defer os.RemoveAll(t.Name()) cnr := cidtest.ID() diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 20c8a946b..7bc84e6c9 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -3,14 +3,15 @@ package engine import ( "errors" "sync" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -20,7 +21,7 @@ type StorageEngine struct { removeDuplicatesInProgress atomic.Bool - mtx *sync.RWMutex + mtx sync.RWMutex shards map[string]hashedShard @@ -35,6 +36,7 @@ type StorageEngine struct { err error } + evacuateLimiter *evacuationLimiter } type shardWrapper struct { @@ -44,6 +46,7 @@ type shardWrapper struct { type setModeRequest struct { sh *shard.Shard + isMeta bool errorCount uint32 } @@ -69,7 +72,7 @@ func (e *StorageEngine) setModeLoop() { if !ok { inProgress[sid] = struct{}{} go func() { - e.moveToDegraded(r.sh, r.errorCount) + e.moveToDegraded(r.sh, r.errorCount, r.isMeta) mtx.Lock() delete(inProgress, sid) @@ -81,34 +84,32 @@ func (e *StorageEngine) setModeLoop() { } } -func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { +func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) { + sid := sh.ID() + log := e.log.With( + zap.Stringer("shard_id", sid), + zap.Uint32("error count", errCount)) + e.mtx.RLock() defer e.mtx.RUnlock() - sid := sh.ID() - err := sh.SetMode(mode.DegradedReadOnly) - if err != nil { - e.log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount), - zap.Error(err)) - - err = sh.SetMode(mode.ReadOnly) - if err != nil { - e.log.Error(logs.EngineFailedToMoveShardInReadonlyMode, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount), - zap.Error(err)) - } else { - e.log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount)) + if isMeta { + err := sh.SetMode(mode.DegradedReadOnly) + if err == nil { + log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) + return } - } else { - e.log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount)) + log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, + zap.Error(err)) } + + err := sh.SetMode(mode.ReadOnly) + if err != nil { + log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) + return + } + + log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } // reportShardErrorBackground increases shard error counter and logs an error. @@ -130,7 +131,8 @@ func (e *StorageEngine) reportShardErrorBackground(id string, msg string, err er return } - errCount := sh.errorCount.Inc() + errCount := sh.errorCount.Add(1) + sh.Shard.IncErrorCounter() e.reportShardErrorWithFlags(sh.Shard, errCount, false, msg, err) } @@ -148,7 +150,8 @@ func (e *StorageEngine) reportShardError( return } - errCount := sh.errorCount.Inc() + errCount := sh.errorCount.Add(1) + sh.Shard.IncErrorCounter() e.reportShardErrorWithFlags(sh.Shard, errCount, true, msg, err, fields...) } @@ -170,11 +173,13 @@ func (e *StorageEngine) reportShardErrorWithFlags( return } + isMeta := errors.As(err, new(metaerr.Error)) if block { - e.moveToDegraded(sh, errCount) + e.moveToDegraded(sh, errCount, isMeta) } else { req := setModeRequest{ errorCount: errCount, + isMeta: isMeta, sh: sh, } @@ -205,6 +210,8 @@ type cfg struct { metrics MetricRegister shardPoolSize uint32 + + lowMem bool } func defaultCfg() *cfg { @@ -224,12 +231,12 @@ func New(opts ...Option) *StorageEngine { } return &StorageEngine{ - cfg: c, - mtx: new(sync.RWMutex), - shards: make(map[string]hashedShard), - shardPools: make(map[string]util.WorkerPool), - closeCh: make(chan struct{}), - setModeCh: make(chan setModeRequest), + cfg: c, + shards: make(map[string]hashedShard), + shardPools: make(map[string]util.WorkerPool), + closeCh: make(chan struct{}), + setModeCh: make(chan setModeRequest), + evacuateLimiter: &evacuationLimiter{}, } } @@ -260,3 +267,10 @@ func WithErrorThreshold(sz uint32) Option { c.errorsThreshold = sz } } + +// WithLowMemoryConsumption returns an option to set the flag to reduce memory consumption by reducing performance. +func WithLowMemoryConsumption(lowMemCons bool) Option { + return func(c *cfg) { + c.lowMem = lowMemCons + } +} diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 4d2ddc100..26de2e5eb 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "sync/atomic" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -17,12 +18,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "git.frostfs.info/TrueCloudLab/hrw" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zaptest" ) @@ -98,10 +98,10 @@ func (te *testEngineWrapper) setInitializedShards(t testing.TB, shards ...*shard te.engine.shards[s.ID().String()] = hashedShard{ shardWrapper: shardWrapper{ - errorCount: atomic.NewUint32(0), + errorCount: new(atomic.Uint32), Shard: s, }, - hash: hrw.Hash([]byte(s.ID().String())), + hash: hrw.StringHash(s.ID().String()), } te.engine.shardPools[s.ID().String()] = pool te.shardIDs = append(te.shardIDs, s.ID()) @@ -148,7 +148,7 @@ func newStorages(root string, smallSize uint64) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1), blobovniczatree.WithPermissions(0700)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < smallSize }, }, @@ -176,7 +176,7 @@ func newTestStorages(root string, smallSize uint64) ([]blobstor.SubStorage, *tes return []blobstor.SubStorage{ { Storage: smallFileStorage, - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < smallSize }, }, diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index dc28d35fa..18bc72d65 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -154,7 +154,7 @@ func TestErrorReporting(t *testing.T) { for i := uint32(0); i < 2; i++ { _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) - checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly) + checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } @@ -219,7 +219,7 @@ func TestBlobstorFailback(t *testing.T) { require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) } - checkShardState(t, te.ng, te.shards[0].id, 1, mode.DegradedReadOnly) + checkShardState(t, te.ng, te.shards[0].id, 2, mode.ReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 761ed24b9..98a3a202d 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -11,24 +12,43 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) -var ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") +var ( + ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") + + evacuationOperationLogField = zap.String("operation", "evacuation") +) // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { shardID []*shard.ID handler func(context.Context, oid.Address, *objectSDK.Object) error ignoreErrors bool + async bool } // EvacuateShardRes represents result of the EvacuateShard operation. type EvacuateShardRes struct { - count int + evacuated *atomic.Uint64 + total *atomic.Uint64 + failed *atomic.Uint64 +} + +// NewEvacuateShardRes creates new EvacuateShardRes instance. +func NewEvacuateShardRes() *EvacuateShardRes { + return &EvacuateShardRes{ + evacuated: new(atomic.Uint64), + total: new(atomic.Uint64), + failed: new(atomic.Uint64), + } } // WithShardIDList sets shard ID. @@ -46,10 +66,52 @@ func (p *EvacuateShardPrm) WithFaultHandler(f func(context.Context, oid.Address, p.handler = f } -// Count returns amount of evacuated objects. +// WithAsync sets flag to run evacuate async. +func (p *EvacuateShardPrm) WithAsync(async bool) { + p.async = async +} + +// Evacuated returns amount of evacuated objects. // Objects for which handler returned no error are also assumed evacuated. -func (p EvacuateShardRes) Count() int { - return p.count +func (p *EvacuateShardRes) Evacuated() uint64 { + if p == nil { + return 0 + } + return p.evacuated.Load() +} + +// Total returns total count objects to evacuate. +func (p *EvacuateShardRes) Total() uint64 { + if p == nil { + return 0 + } + return p.total.Load() +} + +// Failed returns count of failed objects to evacuate. +func (p *EvacuateShardRes) Failed() uint64 { + if p == nil { + return 0 + } + return p.failed.Load() +} + +// DeepCopy returns deep copy of result instance. +func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { + if p == nil { + return nil + } + + res := &EvacuateShardRes{ + evacuated: new(atomic.Uint64), + total: new(atomic.Uint64), + failed: new(atomic.Uint64), + } + + res.evacuated.Store(p.evacuated.Load()) + res.total.Store(p.total.Load()) + res.failed.Store(p.failed.Load()) + return res } const defaultEvacuateBatchSize = 100 @@ -63,15 +125,29 @@ var errMustHaveTwoShards = errors.New("must have at least 1 spare shard") // Evacuate moves data from one shard to the others. // The shard being moved must be in read-only mode. -func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (EvacuateShardRes, error) { +func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*EvacuateShardRes, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + shardIDs := make([]string, len(prm.shardID)) for i := range prm.shardID { shardIDs[i] = prm.shardID[i].String() } + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Evacuate", + trace.WithAttributes( + attribute.StringSlice("shardIDs", shardIDs), + attribute.Bool("async", prm.async), + attribute.Bool("ignoreErrors", prm.ignoreErrors), + )) + defer span.End() + shards, weights, err := e.getActualShards(shardIDs, prm.handler != nil) if err != nil { - return EvacuateShardRes{}, err + return nil, err } shardsToEvacuate := make(map[string]*shard.Shard) @@ -83,23 +159,97 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (Eva } } - e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs)) + res := NewEvacuateShardRes() + ctx = ctxOrBackground(ctx, prm.async) + eg, egCtx, err := e.evacuateLimiter.TryStart(ctx, shardIDs, res) - var res EvacuateShardRes + if err != nil { + return nil, err + } + + eg.Go(func() error { + return e.evacuateShards(egCtx, shardIDs, prm, res, shards, weights, shardsToEvacuate) + }) + + if prm.async { + return nil, nil + } + + return res, eg.Wait() +} + +func ctxOrBackground(ctx context.Context, background bool) context.Context { + if background { + return context.Background() + } + return ctx +} + +func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, prm EvacuateShardPrm, res *EvacuateShardRes, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + var err error + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShards", + trace.WithAttributes( + attribute.StringSlice("shardIDs", shardIDs), + attribute.Bool("async", prm.async), + attribute.Bool("ignoreErrors", prm.ignoreErrors), + )) + + defer func() { + span.End() + e.evacuateLimiter.Complete(err) + }() + + e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField) + + err = e.getTotalObjectsCount(ctx, shardsToEvacuate, res) + if err != nil { + e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField) + return err + } for _, shardID := range shardIDs { - if err = e.evacuateShard(ctx, shardID, prm, &res, shards, weights, shardsToEvacuate); err != nil { - e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs)) - return res, err + if err = e.evacuateShard(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { + e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField) + return err } } - e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, zap.Strings("shard_ids", shardIDs)) - return res, nil + e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, + zap.Strings("shard_ids", shardIDs), + evacuationOperationLogField, + zap.Uint64("total", res.Total()), + zap.Uint64("evacuated", res.Evacuated()), + zap.Uint64("failed", res.Failed()), + ) + return nil +} + +func (e *StorageEngine) getTotalObjectsCount(ctx context.Context, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.getTotalObjectsCount") + defer span.End() + + for _, sh := range shardsToEvacuate { + cnt, err := sh.LogicalObjectsCount(ctx) + if err != nil { + if errors.Is(err, shard.ErrDegradedMode) { + continue + } + return err + } + res.total.Add(cnt) + } + return nil } func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShard", + trace.WithAttributes( + attribute.String("shardID", shardID), + )) + defer span.End() + var listPrm shard.ListWithCursorPrm listPrm.WithCount(defaultEvacuateBatchSize) @@ -111,11 +261,12 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E // TODO (@fyrchik): #1731 this approach doesn't work in degraded modes // because ListWithCursor works only with the metabase. - listRes, err := sh.ListWithCursor(listPrm) + listRes, err := sh.ListWithCursor(ctx, listPrm) if err != nil { if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { break } + e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField) return err } @@ -168,6 +319,12 @@ func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, toEvacuate []object.AddressWithType, prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateObjects", + trace.WithAttributes( + attribute.Int("objects_count", len(toEvacuate)), + )) + defer span.End() + for i := range toEvacuate { select { case <-ctx.Done(): @@ -182,12 +339,14 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to getRes, err := sh.Get(ctx, getPrm) if err != nil { if prm.ignoreErrors { + res.failed.Add(1) continue } + e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) return err } - evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), sh, res, shards, weights, shardsToEvacuate) + evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), sh, shards, weights, shardsToEvacuate, res) if err != nil { return err } @@ -204,16 +363,17 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to err = prm.handler(ctx, addr, getRes.Object()) if err != nil { + e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) return err } - res.count++ + res.evacuated.Add(1) } return nil } -func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) (bool, error) { - hrw.SortHasherSliceByWeightValue(shards, weights, hrw.Hash([]byte(addr.EncodeToString()))) +func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) (bool, error) { + hrw.SortHasherSliceByWeightValue(shards, weights, hrw.StringHash(addr.EncodeToString())) for j := range shards { select { case <-ctx.Done(): @@ -227,11 +387,12 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add putDone, exists := e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object) if putDone || exists { if putDone { + res.evacuated.Add(1) e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), - zap.Stringer("addr", addr)) - res.count++ + zap.Stringer("addr", addr), + evacuationOperationLogField) } return true, nil } @@ -239,3 +400,23 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add return false, nil } + +func (e *StorageEngine) GetEvacuationState(ctx context.Context) (*EvacuationState, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + return e.evacuateLimiter.GetState(), nil +} + +func (e *StorageEngine) EnqueRunningEvacuationStop(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + return e.evacuateLimiter.CancelIfRunning() +} diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go new file mode 100644 index 000000000..62795fa1a --- /dev/null +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -0,0 +1,178 @@ +package engine + +import ( + "context" + "fmt" + "sync" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "golang.org/x/sync/errgroup" +) + +type EvacuateProcessState int + +const ( + EvacuateProcessStateUndefined EvacuateProcessState = iota + EvacuateProcessStateRunning + EvacuateProcessStateCompleted +) + +type EvacuationState struct { + shardIDs []string + processState EvacuateProcessState + startedAt time.Time + finishedAt time.Time + result *EvacuateShardRes + errMessage string +} + +func (s *EvacuationState) ShardIDs() []string { + if s == nil { + return nil + } + return s.shardIDs +} + +func (s *EvacuationState) Evacuated() uint64 { + if s == nil { + return 0 + } + return s.result.Evacuated() +} + +func (s *EvacuationState) Total() uint64 { + if s == nil { + return 0 + } + return s.result.Total() +} + +func (s *EvacuationState) Failed() uint64 { + if s == nil { + return 0 + } + return s.result.Failed() +} + +func (s *EvacuationState) ProcessingStatus() EvacuateProcessState { + if s == nil { + return EvacuateProcessStateUndefined + } + return s.processState +} + +func (s *EvacuationState) StartedAt() *time.Time { + if s == nil { + return nil + } + defaultTime := time.Time{} + if s.startedAt == defaultTime { + return nil + } + return &s.startedAt +} + +func (s *EvacuationState) FinishedAt() *time.Time { + if s == nil { + return nil + } + defaultTime := time.Time{} + if s.finishedAt == defaultTime { + return nil + } + return &s.finishedAt +} + +func (s *EvacuationState) ErrorMessage() string { + if s == nil { + return "" + } + return s.errMessage +} + +func (s *EvacuationState) DeepCopy() *EvacuationState { + if s == nil { + return nil + } + shardIDs := make([]string, len(s.shardIDs)) + copy(shardIDs, s.shardIDs) + + return &EvacuationState{ + shardIDs: shardIDs, + processState: s.processState, + startedAt: s.startedAt, + finishedAt: s.finishedAt, + errMessage: s.errMessage, + result: s.result.DeepCopy(), + } +} + +type evacuationLimiter struct { + state EvacuationState + eg *errgroup.Group + cancel context.CancelFunc + + guard sync.RWMutex +} + +func (l *evacuationLimiter) TryStart(ctx context.Context, shardIDs []string, result *EvacuateShardRes) (*errgroup.Group, context.Context, error) { + l.guard.Lock() + defer l.guard.Unlock() + + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + default: + } + + if l.state.processState == EvacuateProcessStateRunning { + return nil, nil, logicerr.New(fmt.Sprintf("evacuate is already running for shard ids %v", l.state.shardIDs)) + } + + var egCtx context.Context + egCtx, l.cancel = context.WithCancel(ctx) + l.eg, egCtx = errgroup.WithContext(egCtx) + l.state = EvacuationState{ + shardIDs: shardIDs, + processState: EvacuateProcessStateRunning, + startedAt: time.Now().UTC(), + result: result, + } + + return l.eg, egCtx, nil +} + +func (l *evacuationLimiter) Complete(err error) { + l.guard.Lock() + defer l.guard.Unlock() + + errMsq := "" + if err != nil { + errMsq = err.Error() + } + l.state.processState = EvacuateProcessStateCompleted + l.state.errMessage = errMsq + l.state.finishedAt = time.Now().UTC() + + l.eg = nil +} + +func (l *evacuationLimiter) GetState() *EvacuationState { + l.guard.RLock() + defer l.guard.RUnlock() + + return l.state.DeepCopy() +} + +func (l *evacuationLimiter) CancelIfRunning() error { + l.guard.Lock() + defer l.guard.Unlock() + + if l.state.processState != EvacuateProcessStateRunning { + return logicerr.New("there is no running evacuation task") + } + + l.cancel() + return nil +} diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index bea6d4ff5..b642ea065 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strconv" "testing" + "time" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -21,12 +22,13 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" + "golang.org/x/sync/errgroup" ) func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEngine, []*shard.ID, []*objectSDK.Object) { dir := t.TempDir() - te := testNewEngine(t, WithShardPoolSize(1)). + te := testNewEngine(t). setShardsNumOpts(t, shardNum, func(id int) []shard.Option { return []shard.Option{ shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), @@ -67,7 +69,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng err := e.Put(context.Background(), putPrm) require.NoError(t, err) - res, err := e.shards[ids[len(ids)-1].String()].List() + res, err := e.shards[ids[len(ids)-1].String()].List(context.Background()) require.NoError(t, err) if len(res.AddressList()) == objPerShard { break @@ -77,6 +79,8 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng } func TestEvacuateShard(t *testing.T) { + t.Parallel() + const objPerShard = 3 e, ids, objects := newEngineEvacuate(t, 3, objPerShard) @@ -101,14 +105,14 @@ func TestEvacuateShard(t *testing.T) { t.Run("must be read-only", func(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, ErrMustBeReadOnly) - require.Equal(t, 0, res.Count()) + require.Equal(t, uint64(0), res.Evacuated()) }) require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly)) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, objPerShard, res.count) + require.Equal(t, uint64(objPerShard), res.Evacuated()) // We check that all objects are available both before and after shard removal. // First case is a real-world use-case. It ensures that an object can be put in presense @@ -119,7 +123,7 @@ func TestEvacuateShard(t *testing.T) { // Calling it again is OK, but all objects are already moved, so no new PUTs should be done. res, err = e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, 0, res.count) + require.Equal(t, uint64(0), res.Evacuated()) checkHasObjects(t) @@ -132,10 +136,12 @@ func TestEvacuateShard(t *testing.T) { } func TestEvacuateNetwork(t *testing.T) { + t.Parallel() + var errReplication = errors.New("handler error") - acceptOneOf := func(objects []*objectSDK.Object, max int) func(context.Context, oid.Address, *objectSDK.Object) error { - var n int + acceptOneOf := func(objects []*objectSDK.Object, max uint64) func(context.Context, oid.Address, *objectSDK.Object) error { + var n uint64 return func(_ context.Context, addr oid.Address, obj *objectSDK.Object) error { if n == max { return errReplication @@ -154,6 +160,7 @@ func TestEvacuateNetwork(t *testing.T) { } t.Run("single shard", func(t *testing.T) { + t.Parallel() e, ids, objects := newEngineEvacuate(t, 1, 3) evacuateShardID := ids[0].String() @@ -164,15 +171,16 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errMustHaveTwoShards) - require.Equal(t, 0, res.Count()) + require.Equal(t, uint64(0), res.Evacuated()) prm.handler = acceptOneOf(objects, 2) res, err = e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, 2, res.Count()) + require.Equal(t, uint64(2), res.Evacuated()) }) t.Run("multiple shards, evacuate one", func(t *testing.T) { + t.Parallel() e, ids, objects := newEngineEvacuate(t, 2, 3) require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) @@ -184,26 +192,27 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, 2, res.Count()) + require.Equal(t, uint64(2), res.Evacuated()) t.Run("no errors", func(t *testing.T) { prm.handler = acceptOneOf(objects, 3) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, 3, res.Count()) + require.Equal(t, uint64(3), res.Evacuated()) }) }) t.Run("multiple shards, evacuate many", func(t *testing.T) { + t.Parallel() e, ids, objects := newEngineEvacuate(t, 4, 5) evacuateIDs := ids[0:3] - var totalCount int + var totalCount uint64 for i := range evacuateIDs { - res, err := e.shards[ids[i].String()].List() + res, err := e.shards[ids[i].String()].List(context.Background()) require.NoError(t, err) - totalCount += len(res.AddressList()) + totalCount += uint64(len(res.AddressList())) } for i := range ids { @@ -216,19 +225,20 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, totalCount-1, res.Count()) + require.Equal(t, totalCount-1, res.Evacuated()) t.Run("no errors", func(t *testing.T) { prm.handler = acceptOneOf(objects, totalCount) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, totalCount, res.Count()) + require.Equal(t, totalCount, res.Evacuated()) }) }) } func TestEvacuateCancellation(t *testing.T) { + t.Parallel() e, ids, _ := newEngineEvacuate(t, 2, 3) require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) @@ -250,5 +260,114 @@ func TestEvacuateCancellation(t *testing.T) { res, err := e.Evacuate(ctx, prm) require.ErrorContains(t, err, "context canceled") - require.Equal(t, 0, res.Count()) + require.Equal(t, uint64(0), res.Evacuated()) +} + +func TestEvacuateSingleProcess(t *testing.T) { + e, ids, _ := newEngineEvacuate(t, 2, 3) + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + + blocker := make(chan interface{}) + running := make(chan interface{}) + + var prm EvacuateShardPrm + prm.shardID = ids[1:2] + prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + select { + case <-running: + default: + close(running) + } + <-blocker + return nil + } + + eg, egCtx := errgroup.WithContext(context.Background()) + eg.Go(func() error { + res, err := e.Evacuate(egCtx, prm) + require.NoError(t, err, "first evacuation failed") + require.Equal(t, uint64(3), res.Evacuated()) + return nil + }) + eg.Go(func() error { + <-running + res, err := e.Evacuate(egCtx, prm) + require.ErrorContains(t, err, "evacuate is already running for shard ids", "second evacuation not failed") + require.Equal(t, uint64(0), res.Evacuated()) + close(blocker) + return nil + }) + require.NoError(t, eg.Wait()) +} + +func TestEvacuateAsync(t *testing.T) { + e, ids, _ := newEngineEvacuate(t, 2, 3) + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + + blocker := make(chan interface{}) + running := make(chan interface{}) + + var prm EvacuateShardPrm + prm.shardID = ids[1:2] + prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + select { + case <-running: + default: + close(running) + } + <-blocker + return nil + } + + st, err := e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get init state failed") + require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state") + require.Equal(t, uint64(0), st.Evacuated(), "invalid init count") + require.Nil(t, st.StartedAt(), "invalid init started at") + require.Nil(t, st.FinishedAt(), "invalid init finished at") + require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid init error message") + + eg, egCtx := errgroup.WithContext(context.Background()) + eg.Go(func() error { + res, err := e.Evacuate(egCtx, prm) + require.NoError(t, err, "first evacuation failed") + require.Equal(t, uint64(3), res.Evacuated()) + return nil + }) + + <-running + + st, err = e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get running state failed") + require.Equal(t, EvacuateProcessStateRunning, st.ProcessingStatus(), "invalid running state") + require.Equal(t, uint64(0), st.Evacuated(), "invalid running count") + require.NotNil(t, st.StartedAt(), "invalid running started at") + require.Nil(t, st.FinishedAt(), "invalid init finished at") + expectedShardIDs := make([]string, 0, 2) + for _, id := range ids[1:2] { + expectedShardIDs = append(expectedShardIDs, id.String()) + } + require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid running shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid init error message") + + close(blocker) + + require.Eventually(t, func() bool { + st, err = e.GetEvacuationState(context.Background()) + return st.ProcessingStatus() == EvacuateProcessStateCompleted + }, 3*time.Second, 10*time.Millisecond, "invalid final state") + + require.NoError(t, err, "get final state failed") + require.Equal(t, uint64(3), st.Evacuated(), "invalid final count") + require.NotNil(t, st.StartedAt(), "invalid final started at") + require.NotNil(t, st.FinishedAt(), "invalid final finished at") + require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid final error message") + + require.NoError(t, eg.Wait()) } diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 683b7bde8..bd094770a 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -4,10 +4,10 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -64,7 +64,7 @@ func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err er func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddGetDuration)() + defer elapsed("Get", e.metrics.AddMethodDuration)() } var errNotFound apistatus.ObjectNotFound diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 130e76c3d..ca51015db 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -4,10 +4,10 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -68,7 +68,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) defer span.End() if e.metrics != nil { - defer elapsed(e.metrics.AddHeadDuration)() + defer elapsed("Head", e.metrics.AddMethodDuration)() } var ( diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index bf00c4289..d5bf8429c 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -8,7 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -17,7 +17,7 @@ func TestHeadRaw(t *testing.T) { defer os.RemoveAll(t.Name()) cnr := cidtest.ID() - splitID := object.NewSplitID() + splitID := objectSDK.NewSplitID() parent := testutil.GenerateObjectWithCID(cnr) testutil.AddAttribute(parent, "foo", "bar") @@ -70,7 +70,7 @@ func TestHeadRaw(t *testing.T) { _, err = e.Head(context.Background(), headPrm) require.Error(t, err) - var si *object.SplitInfoError + var si *objectSDK.SplitInfoError require.ErrorAs(t, err, &si) // SplitInfoError should contain info from both shards diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index b1204ed99..0b9ae602b 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -4,10 +4,10 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -78,7 +78,7 @@ func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRe func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddInhumeDuration)() + defer elapsed("Inhume", e.metrics.AddMethodDuration)() } var shPrm shard.InhumePrm diff --git a/pkg/local_object_storage/engine/list.go b/pkg/local_object_storage/engine/list.go index 8781416f4..f9229a2b1 100644 --- a/pkg/local_object_storage/engine/list.go +++ b/pkg/local_object_storage/engine/list.go @@ -1,6 +1,7 @@ package engine import ( + "context" "math/rand" "sort" @@ -96,7 +97,7 @@ func (l ListWithCursorRes) Cursor() *Cursor { // // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. -func (e *StorageEngine) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) { +func (e *StorageEngine) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (ListWithCursorRes, error) { result := make([]objectcore.AddressWithType, 0, prm.count) // Set initial cursors @@ -142,7 +143,7 @@ func (e *StorageEngine) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes shardPrm.WithCount(count) shardPrm.WithCursor(cursor.getCurrentShardCursor()) - res, err := shardInstance.ListWithCursor(shardPrm) + res, err := shardInstance.ListWithCursor(ctx, shardPrm) if err != nil { cursor.setShardRead(curr) continue diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 44062be68..6cea2d0f4 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -29,6 +29,8 @@ func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithTy } func TestListWithCursor(t *testing.T) { + t.Parallel() + tests := []struct { name string shardNum int @@ -60,8 +62,10 @@ func TestListWithCursor(t *testing.T) { batchSize: 100, }, } - for _, tt := range tests { + for i := range tests { + tt := tests[i] t.Run(tt.name, func(t *testing.T) { + t.Parallel() e := testNewEngine(t).setShardsNumOpts(t, tt.shardNum, func(id int) []shard.Option { return []shard.Option{ shard.WithLogger(&logger.Logger{Logger: zap.L()}), @@ -103,7 +107,7 @@ func TestListWithCursor(t *testing.T) { var prm ListWithCursorPrm prm.count = tt.batchSize for { - res, err := e.ListWithCursor(prm) + res, err := e.ListWithCursor(context.Background(), prm) if err == ErrEndOfListing { require.Empty(t, res.AddressList()) break diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 4562c1a57..61a5a0dc9 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -4,9 +4,9 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 4c89b9226..a7f0a1552 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -15,7 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/panjf2000/ants/v2" @@ -31,6 +31,8 @@ func (t tss) IsTombstoneAvailable(ctx context.Context, _ oid.Address, epoch uint } func TestLockUserScenario(t *testing.T) { + t.Parallel() + // Tested user actions: // 1. stores some object // 2. locks the object @@ -81,7 +83,7 @@ func TestLockUserScenario(t *testing.T) { lockerAddr.SetContainer(cnr) lockerAddr.SetObject(lockerID) - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(strconv.Itoa(lockerExpiresAfter)) @@ -103,9 +105,9 @@ func TestLockUserScenario(t *testing.T) { require.NoError(t, err) // 2. - var locker object.Lock + var locker objectSDK.Lock locker.WriteMembers([]oid.ID{id}) - object.WriteLock(lockerObj, locker) + objectSDK.WriteLock(lockerObj, locker) err = Put(context.Background(), e, lockerObj) require.NoError(t, err) @@ -121,7 +123,7 @@ func TestLockUserScenario(t *testing.T) { require.ErrorAs(t, err, new(apistatus.ObjectLocked)) // 4. - tombObj.SetType(object.TypeTombstone) + tombObj.SetType(objectSDK.TypeTombstone) tombObj.SetID(tombForLockID) tombObj.SetAttributes(a) @@ -146,6 +148,8 @@ func TestLockUserScenario(t *testing.T) { } func TestLockExpiration(t *testing.T) { + t.Parallel() + // Tested scenario: // 1. some object is stored // 2. lock object for it is stored, and the object is locked @@ -184,12 +188,12 @@ func TestLockExpiration(t *testing.T) { require.NoError(t, err) // 2. - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(strconv.Itoa(lockerExpiresAfter)) lock := testutil.GenerateObjectWithCID(cnr) - lock.SetType(object.TypeLock) + lock.SetType(objectSDK.TypeLock) lock.SetAttributes(a) err = Put(context.Background(), e, lock) @@ -222,6 +226,8 @@ func TestLockExpiration(t *testing.T) { } func TestLockForceRemoval(t *testing.T) { + t.Parallel() + // Tested scenario: // 1. some object is stored // 2. lock object for it is stored, and the object is locked @@ -260,7 +266,7 @@ func TestLockForceRemoval(t *testing.T) { // 2. lock := testutil.GenerateObjectWithCID(cnr) - lock.SetType(object.TypeLock) + lock.SetType(objectSDK.TypeLock) err = Put(context.Background(), e, lock) require.NoError(t, err) diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 13dcdfe02..fcac2dc60 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -2,34 +2,54 @@ package engine import ( "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) type MetricRegister interface { - AddListContainersDuration(d time.Duration) - AddEstimateContainerSizeDuration(d time.Duration) - AddDeleteDuration(d time.Duration) - AddExistsDuration(d time.Duration) - AddGetDuration(d time.Duration) - AddHeadDuration(d time.Duration) - AddInhumeDuration(d time.Duration) - AddPutDuration(d time.Duration) - AddRangeDuration(d time.Duration) - AddSearchDuration(d time.Duration) - AddListObjectsDuration(d time.Duration) + AddMethodDuration(method string, d time.Duration) SetObjectCounter(shardID, objectType string, v uint64) AddToObjectCounter(shardID, objectType string, delta int) - SetReadonly(shardID string, readonly bool) + SetMode(shardID string, mode mode.Mode) AddToContainerSize(cnrID string, size int64) AddToPayloadCounter(shardID string, size int64) + IncErrorCounter(shardID string) + ClearErrorCounter(shardID string) + DeleteShardMetrics(shardID string) + + WriteCache() metrics.WriteCacheMetrics + GC() metrics.GCMetrics } -func elapsed(addFunc func(d time.Duration)) func() { +func elapsed(method string, addFunc func(method string, d time.Duration)) func() { t := time.Now() return func() { - addFunc(time.Since(t)) + addFunc(method, time.Since(t)) } } + +type gcMetrics struct { + storage metrics.GCMetrics + shardID string +} + +func (m *gcMetrics) AddRunDuration(d time.Duration, success bool) { + m.storage.AddRunDuration(m.shardID, d, success) +} + +func (m *gcMetrics) AddDeletedCount(deleted, failed uint64) { + m.storage.AddDeletedCount(m.shardID, deleted, failed) +} + +func (m *gcMetrics) AddExpiredObjectCollectionDuration(d time.Duration, success bool, objectType string) { + m.storage.AddExpiredObjectCollectionDuration(m.shardID, d, success, objectType) +} + +func (m *gcMetrics) AddInhumedObjectCount(count uint64, objectType string) { + m.storage.AddInhumedObjectCount(m.shardID, count, objectType) +} diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 0543f9f15..2f96b8296 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -4,13 +4,13 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" @@ -57,7 +57,7 @@ func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) { func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { if e.metrics != nil { - defer elapsed(e.metrics.AddPutDuration)() + defer elapsed("Put", e.metrics.AddMethodDuration)() } addr := object.AddressOf(prm.obj) @@ -154,6 +154,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, putSuccess = true }); err != nil { + e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Error(err)) close(exitCh) } diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 3d119ac6f..328df4587 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -5,10 +5,10 @@ import ( "errors" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -80,7 +80,7 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error defer span.End() if e.metrics != nil { - defer elapsed(e.metrics.AddRangeDuration)() + defer elapsed("GetRange", e.metrics.AddMethodDuration)() } var errNotFound apistatus.ObjectNotFound diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index 1ea569928..4b48d179c 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -69,7 +69,7 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat var listPrm shard.ListWithCursorPrm listPrm.WithCount(uint32(prm.Concurrency)) listPrm.WithCursor(cursor) - res, err := sh.ListWithCursor(listPrm) + res, err := sh.ListWithCursor(ctx, listPrm) if err != nil { if errors.Is(err, meta.ErrEndOfListing) { return nil @@ -110,7 +110,7 @@ func (e *StorageEngine) removeObjects(ctx context.Context, ch <-chan oid.Address } for addr := range ch { - h := hrw.Hash([]byte(addr.EncodeToString())) + h := hrw.StringHash(addr.EncodeToString()) shards := sortShardsByWeight(shards, h) found := false for i := range shards { diff --git a/pkg/local_object_storage/engine/remove_copies_test.go b/pkg/local_object_storage/engine/remove_copies_test.go index c53e03bbf..8131fcf0d 100644 --- a/pkg/local_object_storage/engine/remove_copies_test.go +++ b/pkg/local_object_storage/engine/remove_copies_test.go @@ -17,6 +17,8 @@ import ( ) func TestRebalance(t *testing.T) { + t.Parallel() + te := newEngineWithErrorThreshold(t, "", 0) const ( @@ -101,6 +103,8 @@ loop: } func TestRebalanceSingleThread(t *testing.T) { + t.Parallel() + te := newEngineWithErrorThreshold(t, "", 0) obj := testutil.GenerateObjectWithCID(cidtest.ID()) diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index e1039ea23..6a8c9fab9 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -3,10 +3,10 @@ package engine import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -15,7 +15,7 @@ import ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters } // SelectRes groups the resulting values of Select operation. @@ -29,7 +29,7 @@ func (p *SelectPrm) WithContainerID(cnr cid.ID) { } // WithFilters is a Select option to set the object filters. -func (p *SelectPrm) WithFilters(fs object.SearchFilters) { +func (p *SelectPrm) WithFilters(fs objectSDK.SearchFilters) { p.filters = fs } @@ -60,7 +60,7 @@ func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRe func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddSearchDuration)() + defer elapsed("Search", e.metrics.AddMethodDuration)() } addrList := make([]oid.Address, 0) @@ -98,18 +98,18 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, // If limit is zero, then returns all available object addresses. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) List(limit uint64) (res SelectRes, err error) { +func (e *StorageEngine) List(ctx context.Context, limit uint64) (res SelectRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.list(limit) + res, err = e.list(ctx, limit) return err }) return } -func (e *StorageEngine) list(limit uint64) (SelectRes, error) { +func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddListObjectsDuration)() + defer elapsed("ListObjects", e.metrics.AddMethodDuration)() } addrList := make([]oid.Address, 0, limit) @@ -118,7 +118,7 @@ func (e *StorageEngine) list(limit uint64) (SelectRes, error) { // consider iterating over shuffled shards e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - res, err := sh.List() // consider limit result of shard iterator + res, err := sh.List(ctx) // consider limit result of shard iterator if err != nil { e.reportShardError(sh, "could not select objects from shard", err) } else { @@ -144,7 +144,7 @@ func (e *StorageEngine) list(limit uint64) (SelectRes, error) { } // Select selects objects from local storage using provided filters. -func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs object.SearchFilters) ([]oid.Address, error) { +func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs objectSDK.SearchFilters) ([]oid.Address, error) { var selectPrm SelectPrm selectPrm.WithContainerID(cnr) selectPrm.WithFilters(fs) @@ -159,8 +159,8 @@ func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs object.S // List returns `limit` available physically storage object addresses in // engine. If limit is zero, then returns all available object addresses. -func List(storage *StorageEngine, limit uint64) ([]oid.Address, error) { - res, err := storage.List(limit) +func List(ctx context.Context, storage *StorageEngine, limit uint64) ([]oid.Address, error) { + res, err := storage.List(ctx, limit) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 64546d9ef..f362e2a03 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -2,16 +2,17 @@ package engine import ( "fmt" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" "github.com/google/uuid" "github.com/panjf2000/ants/v2" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -49,8 +50,8 @@ func (m *metricsWithID) DecObjectCounter(objectType string) { m.mw.AddToObjectCounter(m.id, objectType, -1) } -func (m *metricsWithID) SetReadonly(readonly bool) { - m.mw.SetReadonly(m.id, readonly) +func (m *metricsWithID) SetMode(mode mode.Mode) { + m.mw.SetMode(m.id, mode) } func (m *metricsWithID) AddToContainerSize(cnr string, size int64) { @@ -61,6 +62,18 @@ func (m *metricsWithID) AddToPayloadSize(size int64) { m.mw.AddToPayloadCounter(m.id, size) } +func (m *metricsWithID) IncErrorCounter() { + m.mw.IncErrorCounter(m.id) +} + +func (m *metricsWithID) ClearErrorCounter() { + m.mw.ClearErrorCounter(m.id) +} + +func (m *metricsWithID) DeleteShardMetrics() { + m.mw.DeleteShardMetrics(m.id) +} + // AddShard adds a new shard to the storage engine. // // Returns any error encountered that did not allow adding a shard. @@ -77,7 +90,7 @@ func (e *StorageEngine) AddShard(opts ...shard.Option) (*shard.ID, error) { } if e.cfg.metrics != nil { - e.cfg.metrics.SetReadonly(sh.ID().String(), sh.GetMode() != mode.ReadWrite) + e.cfg.metrics.SetMode(sh.ID().String(), sh.GetMode()) } return sh.ID(), nil @@ -89,18 +102,7 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { return nil, fmt.Errorf("could not generate shard ID: %w", err) } - e.mtx.RLock() - - if e.metrics != nil { - opts = append(opts, shard.WithMetricsWriter( - &metricsWithID{ - id: id.String(), - mw: e.metrics, - }, - )) - } - - e.mtx.RUnlock() + opts = e.appendMetrics(id, opts) sh := shard.New(append(opts, shard.WithID(id), @@ -117,6 +119,36 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { return sh, err } +func (e *StorageEngine) appendMetrics(id *shard.ID, opts []shard.Option) []shard.Option { + e.mtx.RLock() + defer e.mtx.RUnlock() + + if e.metrics != nil { + opts = append(opts, + shard.WithMetricsWriter( + &metricsWithID{ + id: id.String(), + mw: e.metrics, + }, + ), + shard.WithExtraWriteCacheOptions(writecache.WithMetrics( + &writeCacheMetrics{ + shardID: id.String(), + metrics: e.metrics.WriteCache(), + }), + ), + shard.WithGCMetrics( + &gcMetrics{ + storage: e.metrics.GC(), + shardID: id.String(), + }, + ), + ) + } + + return opts +} + func (e *StorageEngine) addShard(sh *shard.Shard) error { e.mtx.Lock() defer e.mtx.Unlock() @@ -133,10 +165,10 @@ func (e *StorageEngine) addShard(sh *shard.Shard) error { e.shards[strID] = hashedShard{ shardWrapper: shardWrapper{ - errorCount: atomic.NewUint32(0), + errorCount: new(atomic.Uint32), Shard: sh, }, - hash: hrw.Hash([]byte(strID)), + hash: hrw.StringHash(strID), } e.shardPools[strID] = pool @@ -160,6 +192,8 @@ func (e *StorageEngine) removeShards(ids ...string) { continue } + sh.DeleteShardMetrics() + ss = append(ss, sh) delete(e.shards, id) @@ -175,7 +209,14 @@ func (e *StorageEngine) removeShards(ids ...string) { e.mtx.Unlock() for _, sh := range ss { - err := sh.Close() + err := sh.SetMode(mode.Disabled) + if err != nil { + e.log.Error(logs.EngineCouldNotChangeShardModeToDisabled, + zap.Stringer("id", sh.ID()), + zap.Error(err), + ) + } + err = sh.Close() if err != nil { e.log.Error(logs.EngineCouldNotCloseRemovedShard, zap.Stringer("id", sh.ID()), @@ -209,7 +250,7 @@ func (e *StorageEngine) sortShardsByWeight(objAddr interface{ EncodeToString() s e.mtx.RLock() defer e.mtx.RUnlock() - h := hrw.Hash([]byte(objAddr.EncodeToString())) + h := hrw.StringHash(objAddr.EncodeToString()) shards := make([]hashedShard, 0, len(e.shards)) for _, sh := range e.shards { shards = append(shards, hashedShard(sh)) @@ -267,6 +308,7 @@ func (e *StorageEngine) SetShardMode(id *shard.ID, m mode.Mode, resetErrorCounte if id.String() == shID { if resetErrorCounter { sh.errorCount.Store(0) + sh.Shard.ClearErrorCounter() } return sh.SetMode(m) } diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index e7d66094c..df3e919ec 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -172,7 +172,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s } // TreeGetChildren implements the pilorama.Forest interface. -func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { +func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetChildren", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -183,7 +183,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree defer span.End() var err error - var nodes []uint64 + var nodes []pilorama.NodeInfo for _, sh := range e.sortShardsByWeight(cid) { nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID) if err != nil { @@ -311,6 +311,22 @@ func (e *StorageEngine) TreeExists(ctx context.Context, cid cidSDK.ID, treeID st return err == nil, err } +func (e *StorageEngine) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeHeight", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + index, lst, err := e.getTreeShard(ctx, cid, treeID) + if err != nil { + return 0, nil + } + return lst[index].TreeHeight(ctx, cid, treeID) +} + // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. func (e *StorageEngine) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeUpdateLastSyncHeight", diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index c2bae9772..f1650b5ae 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -8,7 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) func BenchmarkTreeVsSearch(b *testing.B) { @@ -37,7 +37,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { b.Fatal(err) } _, err = te.ng.TreeAddByPath(context.Background(), d, treeID, pilorama.AttributeFilename, nil, - []pilorama.KeyValue{{pilorama.AttributeFilename, []byte(strconv.Itoa(i))}}) + []pilorama.KeyValue{{Key: pilorama.AttributeFilename, Value: []byte(strconv.Itoa(i))}}) if err != nil { b.Fatal(err) } @@ -47,8 +47,8 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { var prm SelectPrm prm.WithContainerID(cid) - var fs object.SearchFilters - fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), object.MatchStringEqual) + var fs objectSDK.SearchFilters + fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), objectSDK.MatchStringEqual) prm.WithFilters(fs) for i := 0; i < b.N; i++ { diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 4effb2b16..2e518c6ff 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -2,9 +2,13 @@ package engine import ( "context" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -52,3 +56,53 @@ func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePr return FlushWriteCacheRes{}, sh.FlushWriteCache(ctx, prm) } + +type writeCacheMetrics struct { + shardID string + metrics metrics.WriteCacheMetrics +} + +func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.StorageType) { + m.metrics.AddMethodDuration(m.shardID, "Get", success, d, st.String()) +} + +func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) { + m.metrics.AddMethodDuration(m.shardID, "Delete", success, d, st.String()) + if success { + m.metrics.DecActualCount(m.shardID, st.String()) + } +} + +func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) { + m.metrics.AddMethodDuration(m.shardID, "Put", success, d, st.String()) + if success { + m.metrics.IncActualCount(m.shardID, st.String()) + } +} + +func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) { + m.metrics.SetEstimateSize(m.shardID, db, writecache.StorageTypeDB.String()) + m.metrics.SetEstimateSize(m.shardID, fstree, writecache.StorageTypeFSTree.String()) +} + +func (m *writeCacheMetrics) SetMode(mode mode.Mode) { + m.metrics.SetMode(m.shardID, mode.String()) +} + +func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) { + m.metrics.SetActualCount(m.shardID, db, writecache.StorageTypeDB.String()) + m.metrics.SetActualCount(m.shardID, fstree, writecache.StorageTypeFSTree.String()) +} + +func (m *writeCacheMetrics) Flush(success bool, st writecache.StorageType) { + m.metrics.IncOperationCounter(m.shardID, "Flush", metrics.NullBool{Bool: success, Valid: true}, st.String()) +} + +func (m *writeCacheMetrics) Evict(st writecache.StorageType) { + m.metrics.DecActualCount(m.shardID, st.String()) + m.metrics.IncOperationCounter(m.shardID, "Evict", metrics.NullBool{}, st.String()) +} + +func (m *writeCacheMetrics) Close() { + m.metrics.Close(m.shardID) +} diff --git a/pkg/local_object_storage/internal/metaerr/error.go b/pkg/local_object_storage/internal/metaerr/error.go new file mode 100644 index 000000000..41b8504bc --- /dev/null +++ b/pkg/local_object_storage/internal/metaerr/error.go @@ -0,0 +1,33 @@ +package metaerr + +import "errors" + +// Error is a wrapper for SSD-related errors. +// In our model it unites metabase, pilorama and write-cache errors. +type Error struct { + err error +} + +// New returns simple error with a provided error message. +func New(msg string) Error { + return Error{err: errors.New(msg)} +} + +// Error implements the error interface. +func (e Error) Error() string { + return e.err.Error() +} + +// Wrap wraps arbitrary error. +// Returns nil if err == nil. +func Wrap(err error) error { + if err != nil { + return Error{err: err} + } + return nil +} + +// Unwrap returns underlying error. +func (e Error) Unwrap() error { + return e.err +} diff --git a/pkg/local_object_storage/internal/metaerr/error_test.go b/pkg/local_object_storage/internal/metaerr/error_test.go new file mode 100644 index 000000000..5a16aa501 --- /dev/null +++ b/pkg/local_object_storage/internal/metaerr/error_test.go @@ -0,0 +1,67 @@ +package metaerr + +import ( + "errors" + "fmt" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestError(t *testing.T) { + t.Run("errors.Is", func(t *testing.T) { + e1 := errors.New("some error") + ee := Wrap(e1) + require.ErrorIs(t, ee, e1) + + e2 := fmt.Errorf("wrap: %w", e1) + ee = Wrap(e2) + require.ErrorIs(t, ee, e1) + require.ErrorIs(t, ee, e2) + + require.Equal(t, errors.Unwrap(ee), e2) + }) + + t.Run("errors.As", func(t *testing.T) { + e1 := testError{42} + ee := Wrap(e1) + + { + var actual testError + require.ErrorAs(t, ee, &actual) + require.Equal(t, e1.data, actual.data) + } + { + var actual Error + require.ErrorAs(t, ee, &actual) + require.Equal(t, e1, actual.err) + } + + e2 := fmt.Errorf("wrap: %w", e1) + ee = Wrap(e2) + + { + var actual testError + require.ErrorAs(t, ee, &actual) + require.Equal(t, e1.data, actual.data) + } + }) +} +func TestNilWrap(t *testing.T) { + require.NoError(t, Wrap(nil)) +} + +func TestErrorMessage(t *testing.T) { + msg := "sth to report" + err := New(msg) + require.Contains(t, err.Error(), msg) +} + +type testError struct { + data uint64 +} + +func (e testError) Error() string { + return strconv.FormatUint(e.data, 10) +} diff --git a/pkg/local_object_storage/internal/testutil/generators.go b/pkg/local_object_storage/internal/testutil/generators.go index 1a1f3cf9e..383c596af 100644 --- a/pkg/local_object_storage/internal/testutil/generators.go +++ b/pkg/local_object_storage/internal/testutil/generators.go @@ -2,13 +2,13 @@ package testutil import ( "encoding/binary" + "sync/atomic" "testing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "golang.org/x/exp/rand" ) @@ -27,7 +27,7 @@ var _ AddressGenerator = &SeqAddrGenerator{} func (g *SeqAddrGenerator) Next() oid.Address { var id oid.ID - binary.LittleEndian.PutUint64(id[:], ((g.cnt.Inc()-1)%g.MaxID)+1) + binary.LittleEndian.PutUint64(id[:], ((g.cnt.Add(1)-1)%g.MaxID)+1) var addr oid.Address addr.SetContainer(cid.ID{}) addr.SetObject(id) @@ -48,7 +48,7 @@ func (g RandAddrGenerator) Next() oid.Address { // ObjectGenerator is the interface of types that generate object entries. type ObjectGenerator interface { - Next() *object.Object + Next() *objectSDK.Object } // SeqObjGenerator is an ObjectGenerator that generates entries with random payloads of size objSize and sequential IDs. @@ -59,7 +59,7 @@ type SeqObjGenerator struct { var _ ObjectGenerator = &SeqObjGenerator{} -func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *object.Object { +func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *objectSDK.Object { data := make([]byte, sz) _, _ = rand.Read(data) obj := GenerateObjectWithCIDWithPayload(cid, data) @@ -67,9 +67,9 @@ func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *ob return obj } -func (g *SeqObjGenerator) Next() *object.Object { +func (g *SeqObjGenerator) Next() *objectSDK.Object { var id oid.ID - binary.LittleEndian.PutUint64(id[:], g.cnt.Inc()) + binary.LittleEndian.PutUint64(id[:], g.cnt.Add(1)) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) } @@ -80,7 +80,7 @@ type RandObjGenerator struct { var _ ObjectGenerator = &RandObjGenerator{} -func (g *RandObjGenerator) Next() *object.Object { +func (g *RandObjGenerator) Next() *objectSDK.Object { var id oid.ID _, _ = rand.Read(id[:]) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) @@ -92,13 +92,13 @@ type OverwriteObjGenerator struct { MaxObjects uint64 } -func (g *OverwriteObjGenerator) Next() *object.Object { +func (g *OverwriteObjGenerator) Next() *objectSDK.Object { var id oid.ID binary.LittleEndian.PutUint64(id[:], uint64(1+rand.Int63n(int64(g.MaxObjects)))) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) } -func AddressFromObject(t testing.TB, obj *object.Object) oid.Address { +func AddressFromObject(t testing.TB, obj *objectSDK.Object) oid.Address { var addr oid.Address id, isSet := obj.ID() diff --git a/pkg/local_object_storage/internal/testutil/object.go b/pkg/local_object_storage/internal/testutil/object.go index 4f6d95816..9cbce27bf 100644 --- a/pkg/local_object_storage/internal/testutil/object.go +++ b/pkg/local_object_storage/internal/testutil/object.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" @@ -16,17 +16,17 @@ import ( const defaultDataSize = 32 -func GenerateObject() *object.Object { +func GenerateObject() *objectSDK.Object { return GenerateObjectWithCID(cidtest.ID()) } -func GenerateObjectWithCID(cnr cid.ID) *object.Object { +func GenerateObjectWithCID(cnr cid.ID) *objectSDK.Object { data := make([]byte, defaultDataSize) _, _ = rand.Read(data) return GenerateObjectWithCIDWithPayload(cnr, data) } -func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *object.Object { +func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *objectSDK.Object { var ver version.Version ver.SetMajor(2) ver.SetMinor(1) @@ -37,7 +37,7 @@ func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *object.Object { var csumTZ checksum.Checksum csumTZ.SetTillichZemor(tz.Sum(csum.Value())) - obj := object.New() + obj := objectSDK.New() obj.SetID(oidtest.ID()) obj.SetOwnerID(usertest.ID()) obj.SetContainerID(cnr) @@ -49,8 +49,8 @@ func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *object.Object { return obj } -func AddAttribute(obj *object.Object, key, val string) { - var attr object.Attribute +func AddAttribute(obj *objectSDK.Object, key, val string) { + var attr objectSDK.Attribute attr.SetKey(key) attr.SetValue(val) @@ -59,7 +59,7 @@ func AddAttribute(obj *object.Object, key, val string) { obj.SetAttributes(attrs...) } -func AddPayload(obj *object.Object, size int) { +func AddPayload(obj *objectSDK.Object, size int) { buf := make([]byte, size) _, _ = rand.Read(buf) diff --git a/pkg/local_object_storage/metabase/children.go b/pkg/local_object_storage/metabase/children.go new file mode 100644 index 000000000..6816358d2 --- /dev/null +++ b/pkg/local_object_storage/metabase/children.go @@ -0,0 +1,78 @@ +package meta + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// GetChildren returns parent -> children map. +// If an object has no children, then map will contain addr -> empty slice value. +func (db *DB) GetChildren(ctx context.Context, addresses []oid.Address) (map[oid.Address][]oid.Address, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("GetChildren", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.GetChildren", + trace.WithAttributes( + attribute.Int("addr_count", len(addresses)), + )) + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return nil, ErrDegradedMode + } + + result := make(map[oid.Address][]oid.Address, len(addresses)) + + buffer := make([]byte, bucketKeySize) + err := db.boltDB.View(func(tx *bbolt.Tx) error { + for _, addr := range addresses { + if _, found := result[addr]; found { + continue + } + + result[addr] = []oid.Address{} + bkt := tx.Bucket(parentBucketName(addr.Container(), buffer)) + if bkt == nil { + continue + } + + binObjIDs, err := decodeList(bkt.Get(objectKey(addr.Object(), buffer))) + if err != nil { + return err + } + + for _, binObjID := range binObjIDs { + var id oid.ID + if err = id.Decode(binObjID); err != nil { + return err + } + var resultAddress oid.Address + resultAddress.SetContainer(addr.Container()) + resultAddress.SetObject(id) + result[addr] = append(result[addr], resultAddress) + } + } + return nil + }) + + if err != nil { + return nil, metaerr.Wrap(err) + } + success = true + return result, nil +} diff --git a/pkg/local_object_storage/metabase/containers.go b/pkg/local_object_storage/metabase/containers.go index 3d69649a9..472b2affc 100644 --- a/pkg/local_object_storage/metabase/containers.go +++ b/pkg/local_object_storage/metabase/containers.go @@ -1,13 +1,28 @@ package meta import ( + "context" "encoding/binary" + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.etcd.io/bbolt" ) -func (db *DB) Containers() (list []cid.ID, err error) { +func (db *DB) Containers(ctx context.Context) (list []cid.ID, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Containers", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.Containers") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -20,8 +35,8 @@ func (db *DB) Containers() (list []cid.ID, err error) { return err }) - - return list, err + success = err == nil + return list, metaerr.Wrap(err) } func (db *DB) containers(tx *bbolt.Tx) ([]cid.ID, error) { @@ -55,7 +70,7 @@ func (db *DB) ContainerSize(id cid.ID) (size uint64, err error) { return err }) - return size, err + return size, metaerr.Wrap(err) } func (db *DB) containerSize(tx *bbolt.Tx, id cid.ID) (uint64, error) { diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index a0be2c743..4e2dd550d 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "math/rand" "sort" "testing" @@ -15,6 +16,8 @@ import ( ) func TestDB_Containers(t *testing.T) { + t.Parallel() + db := newDB(t) const N = 10 @@ -32,7 +35,7 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, err) } - lst, err := db.Containers() + lst, err := db.Containers(context.Background()) require.NoError(t, err) for _, cnr := range lst { @@ -58,7 +61,7 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, putBig(db, obj)) - cnrs, err := db.Containers() + cnrs, err := db.Containers(context.Background()) require.NoError(t, err) cnr, _ := obj.ContainerID() @@ -66,7 +69,7 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, metaInhume(db, object.AddressOf(obj), oidtest.Address())) - cnrs, err = db.Containers() + cnrs, err = db.Containers(context.Background()) require.NoError(t, err) assertContains(cnrs, cnr) }) @@ -76,20 +79,22 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, putBig(db, obj)) - cnrs, err := db.Containers() + cnrs, err := db.Containers(context.Background()) require.NoError(t, err) cnr, _ := obj.ContainerID() assertContains(cnrs, cnr) require.NoError(t, metaToMoveIt(db, object.AddressOf(obj))) - cnrs, err = db.Containers() + cnrs, err = db.Containers(context.Background()) require.NoError(t, err) assertContains(cnrs, cnr) }) } func TestDB_ContainersCount(t *testing.T) { + t.Parallel() + db := newDB(t) const R, T, SG, L = 10, 11, 12, 13 // amount of object per type @@ -122,7 +127,7 @@ func TestDB_ContainersCount(t *testing.T) { return expected[i].EncodeToString() < expected[j].EncodeToString() }) - got, err := db.Containers() + got, err := db.Containers(context.Background()) require.NoError(t, err) sort.Slice(got, func(i, j int) bool { @@ -133,6 +138,8 @@ func TestDB_ContainersCount(t *testing.T) { } func TestDB_ContainerSize(t *testing.T) { + t.Parallel() + db := newDB(t) const ( diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 4ae802aaa..d0a9c4723 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -6,6 +6,7 @@ import ( "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -34,7 +35,7 @@ func (db *DB) Open(readOnly bool) error { } db.boltOptions.ReadOnly = readOnly - return db.openBolt() + return metaerr.Wrap(db.openBolt()) } func (db *DB) openBolt() error { @@ -79,7 +80,7 @@ func (db *DB) openBolt() error { // Does nothing if metabase has already been initialized and filled. To roll back the database to its initial state, // use Reset. func (db *DB) Init() error { - return db.init(false) + return metaerr.Wrap(db.init(false)) } // Reset resets metabase. Works similar to Init but cleans up all static buckets and @@ -92,7 +93,7 @@ func (db *DB) Reset() error { return ErrDegradedMode } - return db.init(true) + return metaerr.Wrap(db.init(true)) } func (db *DB) init(reset bool) error { @@ -167,17 +168,21 @@ func (db *DB) SyncCounters() error { return ErrReadOnlyMode } - return db.boltDB.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { return syncCounter(tx, true) - }) + })) } // Close closes boltDB instance. func (db *DB) Close() error { + var err error if db.boltDB != nil { - return db.boltDB.Close() + err = metaerr.Wrap(db.boltDB.Close()) } - return nil + if err == nil { + db.metrics.Close() + } + return err } // Reload reloads part of the configuration. @@ -201,12 +206,14 @@ func (db *DB) Reload(opts ...Option) (bool, error) { } db.mode = mode.Degraded + db.metrics.SetMode(mode.Degraded) db.info.Path = c.info.Path if err := db.openBolt(); err != nil { - return false, fmt.Errorf("%w: %v", ErrDegradedMode, err) + return false, metaerr.Wrap(fmt.Errorf("%w: %v", ErrDegradedMode, err)) } db.mode = mode.ReadWrite + db.metrics.SetMode(mode.ReadWrite) return true, nil } diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index a07328026..c0dc7886e 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -67,7 +68,7 @@ func (db *DB) ObjectCounters() (cc ObjectCounters, err error) { return nil }) - return + return cc, metaerr.Wrap(err) } // updateCounter updates the object counter. Tx MUST be writable. diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 17a593b6d..2c5fa0233 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -2,12 +2,13 @@ package meta_test import ( "context" + "os" "testing" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -16,20 +17,24 @@ import ( const objCount = 10 func TestCounters(t *testing.T) { - db := newDB(t) - - var c meta.ObjectCounters - var err error + t.Parallel() + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(t.Name())) + }) t.Run("defaults", func(t *testing.T) { - c, err = db.ObjectCounters() + t.Parallel() + db := newDB(t) + c, err := db.ObjectCounters() require.NoError(t, err) require.Zero(t, c.Phy()) require.Zero(t, c.Logic()) }) t.Run("put", func(t *testing.T) { - oo := make([]*object.Object, 0, objCount) + t.Parallel() + db := newDB(t) + oo := make([]*objectSDK.Object, 0, objCount) for i := 0; i < objCount; i++ { oo = append(oo, testutil.GenerateObject()) } @@ -39,10 +44,10 @@ func TestCounters(t *testing.T) { for i := 0; i < objCount; i++ { prm.SetObject(oo[i]) - _, err = db.Put(context.Background(), prm) + _, err := db.Put(context.Background(), prm) require.NoError(t, err) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(i+1), c.Phy()) @@ -50,9 +55,9 @@ func TestCounters(t *testing.T) { } }) - require.NoError(t, db.Reset()) - t.Run("delete", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, false) var prm meta.DeletePrm @@ -63,7 +68,7 @@ func TestCounters(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(1), res.AvailableObjectsRemoved()) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(i), c.Phy()) @@ -71,9 +76,9 @@ func TestCounters(t *testing.T) { } }) - require.NoError(t, db.Reset()) - t.Run("inhume", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, false) inhumedObjs := make([]oid.Address, objCount/2) @@ -94,16 +99,16 @@ func TestCounters(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(len(inhumedObjs)), res.AvailableInhumed()) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(objCount), c.Phy()) require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic()) }) - require.NoError(t, db.Reset()) - t.Run("put_split", func(t *testing.T) { + t.Parallel() + db := newDB(t) parObj := testutil.GenerateObject() // put objects and check that parent info @@ -116,16 +121,16 @@ func TestCounters(t *testing.T) { require.NoError(t, putBig(db, o)) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(i+1), c.Phy()) require.Equal(t, uint64(i+1), c.Logic()) } }) - require.NoError(t, db.Reset()) - t.Run("delete_split", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, true) // delete objects that have parent info @@ -141,9 +146,9 @@ func TestCounters(t *testing.T) { } }) - require.NoError(t, db.Reset()) - t.Run("inhume_split", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, true) inhumedObjs := make([]oid.Address, objCount/2) @@ -160,10 +165,10 @@ func TestCounters(t *testing.T) { prm.SetTombstoneAddress(oidtest.Address()) prm.SetAddresses(inhumedObjs...) - _, err = db.Inhume(context.Background(), prm) + _, err := db.Inhume(context.Background(), prm) require.NoError(t, err) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(objCount), c.Phy()) @@ -186,7 +191,7 @@ func TestCounters_Expired(t *testing.T) { oo := make([]oid.Address, objCount) for i := range oo { - oo[i] = putWithExpiration(t, db, object.TypeRegular, epoch+1) + oo[i] = putWithExpiration(t, db, objectSDK.TypeRegular, epoch+1) } // 1. objects are available and counters are correct @@ -270,12 +275,12 @@ func TestCounters_Expired(t *testing.T) { require.Equal(t, uint64(len(oo)), c.Logic()) } -func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*object.Object { +func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*objectSDK.Object { var prm meta.PutPrm var err error parent := testutil.GenerateObject() - oo := make([]*object.Object, 0, count) + oo := make([]*objectSDK.Object, 0, count) for i := 0; i < count; i++ { o := testutil.GenerateObject() if withParent { diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index 5a9ca3aa9..f5341ff2e 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -14,7 +14,7 @@ import ( v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/mr-tron/base58" "go.etcd.io/bbolt" "go.uber.org/zap" @@ -39,7 +39,7 @@ type DB struct { modeMtx sync.RWMutex mode mode.Mode - matchers map[object.SearchMatchType]matcher + matchers map[objectSDK.SearchMatchType]matcher boltDB *bbolt.DB @@ -60,6 +60,7 @@ type cfg struct { log *logger.Logger epochState EpochState + metrics Metrics } func defaultCfg() *cfg { @@ -70,6 +71,7 @@ func defaultCfg() *cfg { boltBatchDelay: bbolt.DefaultMaxBatchDelay, boltBatchSize: bbolt.DefaultMaxBatchSize, log: &logger.Logger{Logger: zap.L()}, + metrics: &noopMetrics{}, } } @@ -87,20 +89,20 @@ func New(opts ...Option) *DB { return &DB{ cfg: c, - matchers: map[object.SearchMatchType]matcher{ - object.MatchUnknown: { + matchers: map[objectSDK.SearchMatchType]matcher{ + objectSDK.MatchUnknown: { matchSlow: unknownMatcher, matchBucket: unknownMatcherBucket, }, - object.MatchStringEqual: { + objectSDK.MatchStringEqual: { matchSlow: stringEqualMatcher, matchBucket: stringEqualMatcherBucket, }, - object.MatchStringNotEqual: { + objectSDK.MatchStringNotEqual: { matchSlow: stringNotEqualMatcher, matchBucket: stringNotEqualMatcherBucket, }, - object.MatchCommonPrefix: { + objectSDK.MatchCommonPrefix: { matchSlow: stringCommonPrefixMatcher, matchBucket: stringCommonPrefixMatcherBucket, }, @@ -274,7 +276,7 @@ func bucketKeyHelper(hdr string, val string) []byte { return v case v2object.FilterHeaderSplitID: - s := object.NewSplitID() + s := objectSDK.NewSplitID() err := s.Parse(val) if err != nil { @@ -292,6 +294,11 @@ func (db *DB) SetLogger(l *logger.Logger) { db.log = l } +// SetParentID sets parent ID to nested components. It is used after the shard ID was generated to use it in logs. +func (db *DB) SetParentID(parentID string) { + db.metrics.SetParentID(parentID) +} + // WithLogger returns option to set logger of DB. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { @@ -349,3 +356,10 @@ func WithEpochState(s EpochState) Option { c.epochState = s } } + +// WithMetrics returns option to specify metrics collector. +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 5340f5d08..d387b3d04 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -5,10 +5,12 @@ import ( "context" "errors" "fmt" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -70,6 +72,14 @@ type referenceCounter map[string]*referenceNumber // Delete removed object records from metabase indexes. func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { + var ( + startedAt = time.Now() + deleted = false + ) + defer func() { + db.metrics.AddMethodDuration("Delete", time.Since(startedAt), deleted) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Delete", trace.WithAttributes( attribute.Int("addr_count", len(prm.addrs)), @@ -97,6 +107,7 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { return err }) if err == nil { + deleted = true for i := range prm.addrs { storagelog.Write(db.log, storagelog.AddressField(prm.addrs[i]), @@ -108,7 +119,7 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { availableRemoved: availableRemoved, sizes: sizes, availableSizes: availableSizes, - }, err + }, metaerr.Wrap(err) } // deleteGroup deletes object from the metabase. Handles removal of the diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index cfd37b0d2..b6e5ea052 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -3,10 +3,12 @@ package meta import ( "context" "fmt" + "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -44,6 +46,14 @@ func (p ExistsRes) Exists() bool { // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Exists", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Exists", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -64,8 +74,8 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err return err }) - - return + success = err == nil + return res, metaerr.Wrap(err) } func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (exists bool, err error) { diff --git a/pkg/local_object_storage/metabase/expired.go b/pkg/local_object_storage/metabase/expired.go index d20fdbfa9..43933d12d 100644 --- a/pkg/local_object_storage/metabase/expired.go +++ b/pkg/local_object_storage/metabase/expired.go @@ -5,17 +5,37 @@ import ( "errors" "fmt" "strconv" + "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // FilterExpired return expired items from addresses. // Address considered expired if metabase does contain information about expiration and // expiration epoch is less than epoch. func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.Address) ([]oid.Address, error) { + var ( + startedAt = time.Now() + success = true + ) + defer func() { + db.metrics.AddMethodDuration("FilterExpired", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.FilterExpired", + trace.WithAttributes( + attribute.String("epoch", strconv.FormatUint(epoch, 10)), + attribute.Int("addr_count", len(addresses)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -65,8 +85,9 @@ func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.A }) if err != nil { - return nil, err + return nil, metaerr.Wrap(err) } + success = true return result, nil } diff --git a/pkg/local_object_storage/metabase/expired_test.go b/pkg/local_object_storage/metabase/expired_test.go index 5755b5cdb..9a6bcc5db 100644 --- a/pkg/local_object_storage/metabase/expired_test.go +++ b/pkg/local_object_storage/metabase/expired_test.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -77,7 +77,7 @@ func TestDB_SelectExpired(t *testing.T) { require.Contains(t, expired, getAddressSafe(t, expiredObj22)) } -func getAddressSafe(t *testing.T, o *object.Object) oid.Address { +func getAddressSafe(t *testing.T, o *objectSDK.Object) oid.Address { cid, set := o.ContainerID() if !set { t.Fatalf("container id required") diff --git a/pkg/local_object_storage/metabase/generic_test.go b/pkg/local_object_storage/metabase/generic_test.go index 227aa9f8d..9d15b6f7a 100644 --- a/pkg/local_object_storage/metabase/generic_test.go +++ b/pkg/local_object_storage/metabase/generic_test.go @@ -10,6 +10,8 @@ import ( ) func TestGeneric(t *testing.T) { + t.Parallel() + defer func() { _ = os.RemoveAll(t.Name()) }() var n int diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index e76b9d4a7..ad35b4c18 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -3,9 +3,11 @@ package meta import ( "context" "fmt" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -51,6 +53,14 @@ func (r GetRes) Header() *objectSDK.Object { // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Get", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Get", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -73,8 +83,8 @@ func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { return err }) - - return + success = err == nil + return res, metaerr.Wrap(err) } func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw bool, currEpoch uint64) (*objectSDK.Object, error) { diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index 393c9f4d0..df9a3d302 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -2,9 +2,13 @@ package meta import ( "bytes" + "context" "errors" "fmt" + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -57,7 +61,18 @@ func (g *GarbageIterationPrm) SetOffset(offset oid.Address) { // // If h returns ErrInterruptIterator, nil returns immediately. // Returns other errors of h directly. -func (db *DB) IterateOverGarbage(p GarbageIterationPrm) error { +func (db *DB) IterateOverGarbage(ctx context.Context, p GarbageIterationPrm) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateOverGarbage", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateOverGarbage") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -65,9 +80,11 @@ func (db *DB) IterateOverGarbage(p GarbageIterationPrm) error { return ErrDegradedMode } - return db.boltDB.View(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateDeletedObj(tx, gcHandler{p.h}, p.offset) - }) + })) + success = err == nil + return err } // TombstonedObject represents descriptor of the @@ -124,7 +141,18 @@ func (g *GraveyardIterationPrm) SetOffset(offset oid.Address) { // // If h returns ErrInterruptIterator, nil returns immediately. // Returns other errors of h directly. -func (db *DB) IterateOverGraveyard(p GraveyardIterationPrm) error { +func (db *DB) IterateOverGraveyard(ctx context.Context, p GraveyardIterationPrm) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateOverGraveyard", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateOverGraveyard") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -132,9 +160,9 @@ func (db *DB) IterateOverGraveyard(p GraveyardIterationPrm) error { return ErrDegradedMode } - return db.boltDB.View(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateDeletedObj(tx, graveyardHandler{p.h}, p.offset) - }) + })) } type kvHandler interface { @@ -231,7 +259,18 @@ func graveFromKV(k, v []byte) (res TombstonedObject, err error) { // graveyard bucket. // // Returns any error appeared during deletion process. -func (db *DB) DropGraves(tss []TombstonedObject) error { +func (db *DB) DropGraves(ctx context.Context, tss []TombstonedObject) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("DropGraves", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.DropGraves") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index 8cd09e3f7..7476608f2 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -23,7 +23,7 @@ func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) { return nil }) - err := db.IterateOverGraveyard(iterGravePRM) + err := db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) require.Zero(t, counter) @@ -33,7 +33,7 @@ func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) require.Zero(t, counter) } @@ -83,7 +83,7 @@ func TestDB_Iterate_OffsetNotFound(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) // the second object would be put after the @@ -99,7 +99,7 @@ func TestDB_Iterate_OffsetNotFound(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) // the third object would be put before the @@ -164,7 +164,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGravePRM) + err = db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) var iterGCPRM meta.GarbageIterationPrm @@ -175,7 +175,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) // objects covered with a tombstone @@ -255,7 +255,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGraveyardPrm) + err = db.IterateOverGraveyard(context.Background(), iterGraveyardPrm) require.NoError(t, err) require.Equal(t, firstIterationSize, counter) require.Equal(t, firstIterationSize, len(gotGraveyard)) @@ -272,7 +272,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGraveyardPrm) + err = db.IterateOverGraveyard(context.Background(), iterGraveyardPrm) require.NoError(t, err) require.Equal(t, len(expectedGraveyard), counter) require.ElementsMatch(t, gotGraveyard, expectedGraveyard) @@ -287,7 +287,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGraveyardPrm) + err = db.IterateOverGraveyard(context.Background(), iterGraveyardPrm) require.NoError(t, err) require.False(t, iWasCalled) } @@ -348,7 +348,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGarbagePrm) + err = db.IterateOverGarbage(context.Background(), iterGarbagePrm) require.NoError(t, err) require.Equal(t, firstIterationSize, counter) require.Equal(t, firstIterationSize, len(gotGarbage)) @@ -363,7 +363,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGarbagePrm) + err = db.IterateOverGarbage(context.Background(), iterGarbagePrm) require.NoError(t, err) require.Equal(t, len(expectedGarbage), counter) require.ElementsMatch(t, gotGarbage, expectedGarbage) @@ -378,7 +378,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGarbagePrm) + err = db.IterateOverGarbage(context.Background(), iterGarbagePrm) require.NoError(t, err) require.False(t, iWasCalled) } @@ -418,11 +418,11 @@ func TestDB_DropGraves(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGravePRM) + err = db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) require.Equal(t, 2, counter) - err = db.DropGraves(buriedTS) + err = db.DropGraves(context.Background(), buriedTS) require.NoError(t, err) counter = 0 @@ -431,7 +431,7 @@ func TestDB_DropGraves(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGravePRM) + err = db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) require.Zero(t, counter) } diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index a6887a33b..7ba5a68a2 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -5,12 +5,14 @@ import ( "context" "errors" "fmt" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -121,6 +123,13 @@ var ErrLockObjectRemoval = logicerr.New("lock object removal") // NOTE: Marks any object with GC mark (despite any prohibitions on operations // with that object) if WithForceGCMark option has been provided. func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Inhume", time.Since(startedAt), success) + }() _, span := tracing.StartSpanFromContext(ctx, "metabase.Inhume") defer span.End() @@ -137,8 +146,8 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err err err = db.boltDB.Update(func(tx *bbolt.Tx) error { return db.inhumeTx(tx, currEpoch, prm, &res) }) - - return + success = err == nil + return res, metaerr.Wrap(err) } func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes) error { @@ -267,7 +276,7 @@ func (db *DB) markAsGC(graveyardBKT, garbageBKT *bbolt.Bucket, key []byte) (bool return false, garbageBKT.Put(key, zeroValue) } -func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Bucket, targetKey []byte, cnr cid.ID, obj *object.Object, res *InhumeRes) error { +func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Bucket, targetKey []byte, cnr cid.ID, obj *objectSDK.Object, res *InhumeRes) error { containerID, _ := obj.ContainerID() if inGraveyardWithKey(targetKey, graveyardBKT, garbageBKT) == 0 { res.availableImhumed++ @@ -276,7 +285,7 @@ func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Buc // if object is stored, and it is regular object then update bucket // with container size estimations - if obj.Type() == object.TypeRegular { + if obj.Type() == objectSDK.TypeRegular { err := changeContainerSize(tx, cnr, obj.PayloadSize(), false) if err != nil { return err diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 4c9dc782c..a1e21ef25 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -1,27 +1,34 @@ package meta import ( + "context" "errors" "fmt" "strconv" + "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ExpiredObject is a descriptor of expired object from DB. type ExpiredObject struct { - typ object.Type + typ objectSDK.Type addr oid.Address } // Type returns type of the expired object. -func (e *ExpiredObject) Type() object.Type { +func (e *ExpiredObject) Type() objectSDK.Type { return e.typ } @@ -43,7 +50,20 @@ var ErrInterruptIterator = logicerr.New("iterator is interrupted") // // If h returns ErrInterruptIterator, nil returns immediately. // Returns other errors of h directly. -func (db *DB) IterateExpired(epoch uint64, h ExpiredObjectHandler) error { +func (db *DB) IterateExpired(ctx context.Context, epoch uint64, h ExpiredObjectHandler) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateExpired", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateExpired", + trace.WithAttributes( + attribute.String("epoch", strconv.FormatUint(epoch, 10)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -51,9 +71,11 @@ func (db *DB) IterateExpired(epoch uint64, h ExpiredObjectHandler) error { return ErrDegradedMode } - return db.boltDB.View(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateExpired(tx, epoch, h) - }) + })) + success = err == nil + return err } func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) error { @@ -72,12 +94,8 @@ func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) return fmt.Errorf("could not parse container ID of expired bucket: %w", err) } - return b.ForEach(func(expKey, _ []byte) error { + return b.ForEachBucket(func(expKey []byte) error { bktExpired := b.Bucket(expKey) - if bktExpired == nil { - return nil - } - expiresAfter, err := strconv.ParseUint(string(expKey), 10, 64) if err != nil { return fmt.Errorf("could not parse expiration epoch: %w", err) @@ -128,7 +146,17 @@ func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) // Returns other errors of h directly. // // Does not modify tss. -func (db *DB) IterateCoveredByTombstones(tss map[string]oid.Address, h func(oid.Address) error) error { +func (db *DB) IterateCoveredByTombstones(ctx context.Context, tss map[string]oid.Address, h func(oid.Address) error) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateCoveredByTombstones", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateCoveredByTombstones") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 69bf2bee5..034a931d2 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -9,7 +9,7 @@ import ( object2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -20,23 +20,23 @@ func TestDB_IterateExpired(t *testing.T) { const epoch = 13 - mAlive := map[object.Type]oid.Address{} - mExpired := map[object.Type]oid.Address{} + mAlive := map[objectSDK.Type]oid.Address{} + mExpired := map[objectSDK.Type]oid.Address{} - for _, typ := range []object.Type{ - object.TypeRegular, - object.TypeTombstone, - object.TypeLock, + for _, typ := range []objectSDK.Type{ + objectSDK.TypeRegular, + objectSDK.TypeTombstone, + objectSDK.TypeLock, } { mAlive[typ] = putWithExpiration(t, db, typ, epoch) mExpired[typ] = putWithExpiration(t, db, typ, epoch-1) } - expiredLocked := putWithExpiration(t, db, object.TypeRegular, epoch-1) + expiredLocked := putWithExpiration(t, db, objectSDK.TypeRegular, epoch-1) require.NoError(t, db.Lock(context.Background(), expiredLocked.Container(), oidtest.ID(), []oid.ID{expiredLocked.Object()})) - err := db.IterateExpired(epoch, func(exp *meta.ExpiredObject) error { + err := db.IterateExpired(context.Background(), epoch, func(exp *meta.ExpiredObject) error { if addr, ok := mAlive[exp.Type()]; ok { require.NotEqual(t, addr, exp.Address()) } @@ -56,7 +56,7 @@ func TestDB_IterateExpired(t *testing.T) { require.Empty(t, mExpired) } -func putWithExpiration(t *testing.T, db *meta.DB, typ object.Type, expiresAt uint64) oid.Address { +func putWithExpiration(t *testing.T, db *meta.DB, typ objectSDK.Type, expiresAt uint64) oid.Address { obj := testutil.GenerateObject() obj.SetType(typ) testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.FormatUint(expiresAt, 10)) @@ -96,7 +96,7 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { ts.EncodeToString(): ts, } - err = db.IterateCoveredByTombstones(tss, func(addr oid.Address) error { + err = db.IterateCoveredByTombstones(context.Background(), tss, func(addr oid.Address) error { handled = append(handled, addr) return nil }) @@ -112,7 +112,7 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { handled = handled[:0] - err = db.IterateCoveredByTombstones(tss, func(addr oid.Address) error { + err = db.IterateCoveredByTombstones(context.Background(), tss, func(addr oid.Address) error { handled = append(handled, addr) return nil }) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 7fe1040bb..37a574a02 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -1,12 +1,19 @@ package meta import ( + "context" + "time" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ErrEndOfListing is returned from object listing with cursor @@ -60,7 +67,21 @@ func (l ListRes) Cursor() *Cursor { // // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. -func (db *DB) ListWithCursor(prm ListPrm) (res ListRes, err error) { +func (db *DB) ListWithCursor(ctx context.Context, prm ListPrm) (res ListRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("ListWithCursor", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.ListWithCursor", + trace.WithAttributes( + attribute.Int("count", prm.count), + attribute.Bool("has_cursor", prm.cursor != nil), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -74,8 +95,8 @@ func (db *DB) ListWithCursor(prm ListPrm) (res ListRes, err error) { res.addrList, res.cursor, err = db.listWithCursor(tx, result, prm.count, prm.cursor) return err }) - - return res, err + success = err == nil + return res, metaerr.Wrap(err) } func (db *DB) listWithCursor(tx *bbolt.Tx, result []objectcore.AddressWithType, count int, cursor *Cursor) ([]objectcore.AddressWithType, *Cursor, error) { @@ -103,15 +124,15 @@ loop: continue } - var objType object.Type + var objType objectSDK.Type switch prefix { case primaryPrefix: - objType = object.TypeRegular + objType = objectSDK.TypeRegular case lockersPrefix: - objType = object.TypeLock + objType = objectSDK.TypeLock case tombstonePrefix: - objType = object.TypeTombstone + objType = objectSDK.TypeTombstone default: continue } @@ -154,7 +175,7 @@ loop: // selectNFromBucket similar to selectAllFromBucket but uses cursor to find // object to start selecting from. Ignores inhumed objects. func selectNFromBucket(bkt *bbolt.Bucket, // main bucket - objType object.Type, // type of the objects stored in the main bucket + objType objectSDK.Type, // type of the objects stored in the main bucket graveyardBkt, garbageBkt *bbolt.Bucket, // cached graveyard buckets cidRaw []byte, // container ID prefix, optimization cnt cid.ID, // container ID diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 07a0d80f8..abb55c9d1 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "errors" "sort" "testing" @@ -51,7 +52,7 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - res, err := db.ListWithCursor(prm) + res, err := db.ListWithCursor(context.Background(), prm) if err != nil { if err != meta.ErrEndOfListing { b.Fatalf("error: %v", err) @@ -66,6 +67,8 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) { } func TestLisObjectsWithCursor(t *testing.T) { + t.Parallel() + db := newDB(t) const ( @@ -159,6 +162,8 @@ func TestLisObjectsWithCursor(t *testing.T) { } func TestAddObjectDuringListingWithCursor(t *testing.T) { + t.Parallel() + db := newDB(t) const total = 5 @@ -221,6 +226,6 @@ func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]objec listPrm.SetCount(count) listPrm.SetCursor(cursor) - r, err := db.ListWithCursor(listPrm) + r, err := db.ListWithCursor(context.Background(), listPrm) return r.AddressList(), r.Cursor(), err } diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 5c3c9720d..50aac223b 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "fmt" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -35,6 +37,14 @@ func bucketNameLockers(idCnr cid.ID, key []byte) []byte { // // Locked list should be unique. Panics if it is empty. func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid.ID) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Lock", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Lock", trace.WithAttributes( attribute.String("container_id", cnr.EncodeToString()), @@ -56,15 +66,20 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. panic("empty locked list") } - // check if all objects are regular + err := db.lockInternal(locked, cnr, locker) + success = err == nil + return err +} + +func (db *DB) lockInternal(locked []oid.ID, cnr cid.ID, locker oid.ID) error { bucketKeysLocked := make([][]byte, len(locked)) for i := range locked { bucketKeysLocked[i] = objectKey(locked[i], make([]byte, objectKeySize)) } key := make([]byte, cidSize) - return db.boltDB.Update(func(tx *bbolt.Tx) error { - if firstIrregularObjectType(tx, cnr, bucketKeysLocked...) != object.TypeRegular { + return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { + if firstIrregularObjectType(tx, cnr, bucketKeysLocked...) != objectSDK.TypeRegular { return logicerr.Wrap(apistatus.LockNonRegularObject{}) } @@ -82,7 +97,6 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. loop: for i := range bucketKeysLocked { - // decode list of already existing lockers exLockers, err = decodeList(bucketLockedContainer.Get(bucketKeysLocked[i])) if err != nil { return fmt.Errorf("decode list of object lockers: %w", err) @@ -94,14 +108,11 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. } } - // update the list of lockers updLockers, err = encodeList(append(exLockers, keyLocker)) if err != nil { - // maybe continue for the best effort? return fmt.Errorf("encode list of object lockers: %w", err) } - // write updated list of lockers err = bucketLockedContainer.Put(bucketKeysLocked[i], updLockers) if err != nil { return fmt.Errorf("update list of object lockers: %w", err) @@ -109,12 +120,20 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. } return nil - }) + })) } // FreeLockedBy unlocks all objects in DB which are locked by lockers. // Returns slice of unlocked object ID's or an error. func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("FreeLockedBy", time.Since(startedAt), success) + }() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -135,8 +154,9 @@ func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) { return nil }); err != nil { - return nil, err + return nil, metaerr.Wrap(err) } + success = true return unlockedObjects, nil } @@ -279,6 +299,14 @@ func (i IsLockedRes) Locked() bool { // // Returns only non-logical errors related to underlying database. func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IsLocked", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IsLocked", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -291,9 +319,10 @@ func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, e if db.mode.NoMetabase() { return res, ErrDegradedMode } - - return res, db.boltDB.View(func(tx *bbolt.Tx) error { + err = metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { res.locked = objectLocked(tx, prm.addr.Container(), prm.addr.Object()) return nil - }) + })) + success = err == nil + return res, err } diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 7b62841dc..b442296fd 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -9,7 +9,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" @@ -17,6 +17,8 @@ import ( ) func TestDB_Lock(t *testing.T) { + t.Parallel() + cnr := cidtest.ID() db := newDB(t) @@ -26,10 +28,10 @@ func TestDB_Lock(t *testing.T) { }) t.Run("(ir)regular", func(t *testing.T) { - for _, typ := range [...]object.Type{ - object.TypeTombstone, - object.TypeLock, - object.TypeRegular, + for _, typ := range [...]objectSDK.Type{ + objectSDK.TypeTombstone, + objectSDK.TypeLock, + objectSDK.TypeRegular, } { obj := objecttest.Object() obj.SetType(typ) @@ -45,7 +47,7 @@ func TestDB_Lock(t *testing.T) { // try to lock it err = db.Lock(context.Background(), cnr, oidtest.ID(), []oid.ID{id}) - if typ == object.TypeRegular { + if typ == objectSDK.TypeRegular { require.NoError(t, err, typ) } else { require.ErrorAs(t, err, &e, typ) @@ -171,12 +173,14 @@ func TestDB_Lock(t *testing.T) { } func TestDB_Lock_Expired(t *testing.T) { + t.Parallel() + es := &epochState{e: 123} db := newDB(t, meta.WithEpochState(es)) // put an object - addr := putWithExpiration(t, db, object.TypeRegular, 124) + addr := putWithExpiration(t, db, objectSDK.TypeRegular, 124) // expire the obj es.e = 125 @@ -192,6 +196,8 @@ func TestDB_Lock_Expired(t *testing.T) { } func TestDB_IsLocked(t *testing.T) { + t.Parallel() + db := newDB(t) // existing and locked objs @@ -236,10 +242,10 @@ func TestDB_IsLocked(t *testing.T) { } // putAndLockObj puts object, returns it and its locker. -func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*object.Object, *object.Object) { +func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*objectSDK.Object, *objectSDK.Object) { cnr := cidtest.ID() - lockedObjs := make([]*object.Object, 0, numOfLockedObjs) + lockedObjs := make([]*objectSDK.Object, 0, numOfLockedObjs) lockedObjIDs := make([]oid.ID, 0, numOfLockedObjs) for i := 0; i < numOfLockedObjs; i++ { @@ -255,7 +261,7 @@ func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*object.Ob lockObj := testutil.GenerateObjectWithCID(cnr) lockID, _ := lockObj.ID() - lockObj.SetType(object.TypeLock) + lockObj.SetType(objectSDK.TypeLock) err := putBig(db, lockObj) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/metrics.go b/pkg/local_object_storage/metabase/metrics.go new file mode 100644 index 000000000..fc971bd81 --- /dev/null +++ b/pkg/local_object_storage/metabase/metrics.go @@ -0,0 +1,23 @@ +package meta + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) + +type Metrics interface { + SetParentID(parentID string) + + SetMode(m mode.Mode) + Close() + + AddMethodDuration(method string, d time.Duration, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetParentID(string) {} +func (m *noopMetrics) SetMode(mode.Mode) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index dd1cdc900..28beca8f3 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -40,5 +40,6 @@ func (db *DB) SetMode(m mode.Mode) error { } db.mode = m + db.metrics.SetMode(m) return nil } diff --git a/pkg/local_object_storage/metabase/movable.go b/pkg/local_object_storage/metabase/movable.go index 412c46393..763e49a5d 100644 --- a/pkg/local_object_storage/metabase/movable.go +++ b/pkg/local_object_storage/metabase/movable.go @@ -4,7 +4,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -76,7 +77,7 @@ func (db *DB) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (res ToMoveItRes, e return toMoveIt.Put(key, zeroValue) }) - return + return res, metaerr.Wrap(err) } // DoNotMove removes `MoveIt` mark from the object. @@ -98,7 +99,7 @@ func (db *DB) DoNotMove(prm DoNotMovePrm) (res DoNotMoveRes, err error) { return toMoveIt.Delete(key) }) - return + return res, metaerr.Wrap(err) } // Movable returns list of marked objects to move into other shard. @@ -121,7 +122,7 @@ func (db *DB) Movable(_ MovablePrm) (MovableRes, error) { }) }) if err != nil { - return MovableRes{}, err + return MovableRes{}, metaerr.Wrap(err) } // we can parse strings to structures in-place, but probably it seems @@ -132,8 +133,8 @@ func (db *DB) Movable(_ MovablePrm) (MovableRes, error) { for i := range strAddrs { err = decodeAddressFromKey(&addrs[i], []byte(strAddrs[i])) if err != nil { - return MovableRes{}, fmt.Errorf("can't parse object address %v: %w", - strAddrs[i], err) + return MovableRes{}, metaerr.Wrap(fmt.Errorf("can't parse object address %v: %w", + strAddrs[i], err)) } } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index bc6520a05..7a24485b3 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -6,11 +6,13 @@ import ( "errors" "fmt" gio "io" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -57,6 +59,14 @@ var ( // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Put", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Put", trace.WithAttributes( attribute.String("address", objectCore.AddressOf(prm.obj).EncodeToString()), @@ -78,12 +88,13 @@ func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { return db.put(tx, prm.obj, prm.id, nil, currEpoch) }) if err == nil { + success = true storagelog.Write(db.log, storagelog.AddressField(objectCore.AddressOf(prm.obj)), storagelog.OpField("metabase PUT")) } - return + return res, metaerr.Wrap(err) } func (db *DB) put(tx *bbolt.Tx, diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index a3a071d19..dbf89c9c8 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -4,6 +4,7 @@ import ( "context" "runtime" "strconv" + "sync/atomic" "testing" "time" @@ -15,7 +16,6 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func prepareObjects(t testing.TB, n int) []*objectSDK.Object { @@ -49,13 +49,15 @@ func BenchmarkPut(b *testing.B) { // Ensure the benchmark is bound by CPU and not waiting batch-delay time. b.SetParallelism(1) - index := atomic.NewInt64(-1) + var index atomic.Int64 + index.Store(-1) + objs := prepareObjects(b, b.N) b.ResetTimer() b.ReportAllocs() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := metaPut(db, objs[index.Inc()], nil); err != nil { + if err := metaPut(db, objs[index.Add(1)], nil); err != nil { b.Fatal(err) } } @@ -65,12 +67,13 @@ func BenchmarkPut(b *testing.B) { db := newDB(b, meta.WithMaxBatchDelay(time.Millisecond*10), meta.WithMaxBatchSize(1)) - index := atomic.NewInt64(-1) + var index atomic.Int64 + index.Store(-1) objs := prepareObjects(b, b.N) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - if err := metaPut(db, objs[index.Inc()], nil); err != nil { + if err := metaPut(db, objs[index.Add(1)], nil); err != nil { b.Fatal(err) } } diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index a4b14f77c..6a7acd7cc 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -6,12 +6,14 @@ import ( "errors" "fmt" "strings" + "time" v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -29,14 +31,14 @@ type ( cnr cid.ID - fastFilters, slowFilters object.SearchFilters + fastFilters, slowFilters objectSDK.SearchFilters } ) // SelectPrm groups the parameters of Select operation. type SelectPrm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters } // SelectRes groups the resulting values of Select operation. @@ -50,7 +52,7 @@ func (p *SelectPrm) SetContainerID(cnr cid.ID) { } // SetFilters is a Select option to set the object filters. -func (p *SelectPrm) SetFilters(fs object.SearchFilters) { +func (p *SelectPrm) SetFilters(fs objectSDK.SearchFilters) { p.filters = fs } @@ -61,6 +63,14 @@ func (r SelectRes) AddressList() []oid.Address { // Select returns list of addresses of objects that match search filters. func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Select", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Select", trace.WithAttributes( attribute.String("container_id", prm.cnr.EncodeToString()), @@ -74,20 +84,21 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err return res, ErrDegradedMode } - if blindlyProcess(prm.filters) { + if checkNonEmpty(prm.filters) { + success = true return res, nil } currEpoch := db.epochState.CurrentEpoch() - return res, db.boltDB.View(func(tx *bbolt.Tx) error { + return res, metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { res.addrList, err = db.selectObjects(tx, prm.cnr, prm.filters, currEpoch) - + success = err == nil return err - }) + })) } -func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, currEpoch uint64) ([]oid.Address, error) { +func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters, currEpoch uint64) ([]oid.Address, error) { group, err := groupFilters(fs) if err != nil { return nil, err @@ -175,7 +186,7 @@ func selectAllFromBucket(tx *bbolt.Tx, name []byte, to map[string]int, fNum int) func (db *DB) selectFastFilter( tx *bbolt.Tx, cnr cid.ID, // container we search on - f object.SearchFilter, // fast filter + f objectSDK.SearchFilter, // fast filter to map[string]int, // resulting cache fNum int, // index of filter ) { @@ -209,7 +220,7 @@ func (db *DB) selectFastFilter( default: // user attribute bucketName := attributeBucketName(cnr, f.Header(), bucketName) - if f.Operation() == object.MatchNotPresent { + if f.Operation() == objectSDK.MatchNotPresent { selectOutsideFKBT(tx, allBucketNames(cnr), bucketName, to, fNum) } else { db.selectFromFKBT(tx, bucketName, f, to, fNum) @@ -233,7 +244,7 @@ func allBucketNames(cnr cid.ID) (names [][]byte) { return } -func bucketNamesForType(cnr cid.ID, mType object.SearchMatchType, typeVal string) (names [][]byte) { +func bucketNamesForType(cnr cid.ID, mType objectSDK.SearchMatchType, typeVal string) (names [][]byte) { appendNames := func(key string) { fns, ok := mBucketNaming[key] if ok { @@ -245,15 +256,15 @@ func bucketNamesForType(cnr cid.ID, mType object.SearchMatchType, typeVal string switch mType { default: - case object.MatchStringNotEqual: + case objectSDK.MatchStringNotEqual: for key := range mBucketNaming { if key != typeVal { appendNames(key) } } - case object.MatchStringEqual: + case objectSDK.MatchStringEqual: appendNames(typeVal) - case object.MatchCommonPrefix: + case objectSDK.MatchCommonPrefix: for key := range mBucketNaming { if strings.HasPrefix(key, typeVal) { appendNames(key) @@ -269,7 +280,7 @@ func bucketNamesForType(cnr cid.ID, mType object.SearchMatchType, typeVal string func (db *DB) selectFromFKBT( tx *bbolt.Tx, name []byte, // fkbt root bucket name - f object.SearchFilter, // filter for operation and value + f objectSDK.SearchFilter, // filter for operation and value to map[string]int, // resulting cache fNum int, // index of filter ) { // @@ -315,12 +326,8 @@ func selectOutsideFKBT( bktExcl := tx.Bucket(name) if bktExcl != nil { - _ = bktExcl.ForEach(func(k, _ []byte) error { + _ = bktExcl.ForEachBucket(func(k []byte) error { exclBktLeaf := bktExcl.Bucket(k) - if exclBktLeaf == nil { - return nil - } - return exclBktLeaf.ForEach(func(k, _ []byte) error { mExcl[string(k)] = struct{}{} @@ -350,7 +357,7 @@ func selectOutsideFKBT( func (db *DB) selectFromList( tx *bbolt.Tx, name []byte, // list root bucket name - f object.SearchFilter, // filter for operation and value + f objectSDK.SearchFilter, // filter for operation and value to map[string]int, // resulting cache fNum int, // index of filter ) { // @@ -365,7 +372,7 @@ func (db *DB) selectFromList( ) switch op := f.Operation(); op { - case object.MatchStringEqual: + case objectSDK.MatchStringEqual: lst, err = decodeList(bkt.Get(bucketKeyHelper(f.Header(), f.Value()))) if err != nil { db.log.Debug(logs.MetabaseCantDecodeListBucketLeaf, zap.String("error", err.Error())) @@ -409,7 +416,7 @@ func (db *DB) selectFromList( // selectObjectID processes objectID filter with in-place optimizations. func (db *DB) selectObjectID( tx *bbolt.Tx, - f object.SearchFilter, + f objectSDK.SearchFilter, cnr cid.ID, to map[string]int, // resulting cache fNum int, // index of filter @@ -429,7 +436,7 @@ func (db *DB) selectObjectID( } switch op := f.Operation(); op { - case object.MatchStringEqual: + case objectSDK.MatchStringEqual: var id oid.ID if err := id.DecodeString(f.Value()); err == nil { appendOID(id) @@ -444,7 +451,7 @@ func (db *DB) selectObjectID( return } - for _, bucketName := range bucketNamesForType(cnr, object.MatchStringNotEqual, "") { + for _, bucketName := range bucketNamesForType(cnr, objectSDK.MatchStringNotEqual, "") { // copy-paste from DB.selectAllFrom bkt := tx.Bucket(bucketName) if bkt == nil { @@ -468,7 +475,7 @@ func (db *DB) selectObjectID( } // matchSlowFilters return true if object header is matched by all slow filters. -func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f object.SearchFilters, currEpoch uint64) bool { +func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.SearchFilters, currEpoch uint64) bool { if len(f) == 0 { return true } @@ -514,10 +521,10 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f object.SearchFi // groupFilters divides filters in two groups: fast and slow. Fast filters // processed by indexes and slow filters processed after by unmarshaling // object headers. -func groupFilters(filters object.SearchFilters) (filterGroup, error) { +func groupFilters(filters objectSDK.SearchFilters) (filterGroup, error) { res := filterGroup{ - fastFilters: make(object.SearchFilters, 0, len(filters)), - slowFilters: make(object.SearchFilters, 0, len(filters)), + fastFilters: make(objectSDK.SearchFilters, 0, len(filters)), + slowFilters: make(objectSDK.SearchFilters, 0, len(filters)), } for i := range filters { @@ -549,15 +556,12 @@ func markAddressInCache(cache map[string]int, fNum int, addr string) { } } -// returns true if query leads to a deliberately empty result. -func blindlyProcess(fs object.SearchFilters) bool { +// Returns true if at least 1 object can satisfy fs. +func checkNonEmpty(fs objectSDK.SearchFilters) bool { for i := range fs { - if fs[i].Operation() == object.MatchNotPresent && isSystemKey(fs[i].Header()) { + if fs[i].Operation() == objectSDK.MatchNotPresent && isSystemKey(fs[i].Header()) { return true } - - // TODO: #1148 check other cases - // e.g. (a == b) && (a != b) } return false @@ -565,6 +569,5 @@ func blindlyProcess(fs object.SearchFilters) bool { // returns true if string key is a reserved system filter key. func isSystemKey(key string) bool { - // FIXME: #1147 version-dependent approach return strings.HasPrefix(key, v2object.ReservedFilterPrefix) } diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index dab4c028d..e107085ab 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -20,6 +20,8 @@ import ( ) func TestDB_SelectUserAttributes(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -142,6 +144,8 @@ func TestDB_SelectUserAttributes(t *testing.T) { } func TestDB_SelectRootPhyParent(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -293,6 +297,8 @@ func TestDB_SelectRootPhyParent(t *testing.T) { } func TestDB_SelectInhume(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -325,6 +331,8 @@ func TestDB_SelectInhume(t *testing.T) { } func TestDB_SelectPayloadHash(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -393,6 +401,8 @@ func TestDB_SelectPayloadHash(t *testing.T) { } func TestDB_SelectWithSlowFilters(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -498,6 +508,8 @@ func TestDB_SelectWithSlowFilters(t *testing.T) { } func TestDB_SelectObjectID(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -611,6 +623,8 @@ func TestDB_SelectObjectID(t *testing.T) { } func TestDB_SelectSplitID(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -665,6 +679,8 @@ func TestDB_SelectSplitID(t *testing.T) { } func TestDB_SelectContainerID(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -750,6 +766,8 @@ func BenchmarkSelect(b *testing.B) { } func TestExpiredObjects(t *testing.T) { + t.Parallel() + db := newDB(t, meta.WithEpochState(epochState{currEpoch})) checkExpiredObjects(t, db, func(exp, nonExp *objectSDK.Object) { diff --git a/pkg/local_object_storage/metabase/shard_id.go b/pkg/local_object_storage/metabase/shard_id.go index fac8a079f..f60a4724d 100644 --- a/pkg/local_object_storage/metabase/shard_id.go +++ b/pkg/local_object_storage/metabase/shard_id.go @@ -1,6 +1,7 @@ package meta import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" ) @@ -28,7 +29,7 @@ func (db *DB) ReadShardID() ([]byte, error) { } return nil }) - return id, err + return id, metaerr.Wrap(err) } // WriteShardID writes shard it to db. @@ -42,11 +43,11 @@ func (db *DB) WriteShardID(id []byte) error { return ErrReadOnlyMode } - return db.boltDB.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { b, err := tx.CreateBucketIfNotExists(shardInfoBucket) if err != nil { return err } return b.Put(shardIDKey, id) - }) + })) } diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index 794879a3f..6ba5a60cb 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -4,7 +4,8 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" @@ -54,7 +55,7 @@ func (db *DB) StorageID(ctx context.Context, prm StorageIDPrm) (res StorageIDRes return err }) - return + return res, metaerr.Wrap(err) } func (db *DB) storageID(tx *bbolt.Tx, addr oid.Address) ([]byte, error) { @@ -113,5 +114,5 @@ func (db *DB) UpdateStorageID(prm UpdateStorageIDPrm) (res UpdateStorageIDRes, e return err }) - return + return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index 5b27cdc87..b3652a680 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -12,6 +12,8 @@ import ( ) func TestDB_StorageID(t *testing.T) { + t.Parallel() + db := newDB(t) raw1 := testutil.GenerateObject() diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index c50fd051f..4e58ec20b 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -6,7 +6,7 @@ import ( "fmt" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -120,7 +120,7 @@ const ( addressKeySize = cidSize + objectKeySize ) -var splitInfoError *object.SplitInfoError // for errors.As comparisons +var splitInfoError *objectSDK.SplitInfoError // for errors.As comparisons func bucketName(cnr cid.ID, prefix byte, key []byte) []byte { key[0] = prefix @@ -221,7 +221,7 @@ func objectKey(obj oid.ID, key []byte) []byte { // if meets irregular object container in objs - returns its type, otherwise returns object.TypeRegular. // // firstIrregularObjectType(tx, cnr, obj) usage allows getting object type. -func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object.Type { +func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) objectSDK.Type { if len(objs) == 0 { panic("empty object list in firstIrregularObjectType") } @@ -229,11 +229,11 @@ func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object var keys [2][1 + cidSize]byte irregularTypeBuckets := [...]struct { - typ object.Type + typ objectSDK.Type name []byte }{ - {object.TypeTombstone, tombstoneBucketName(idCnr, keys[0][:])}, - {object.TypeLock, bucketNameLockers(idCnr, keys[1][:])}, + {objectSDK.TypeTombstone, tombstoneBucketName(idCnr, keys[0][:])}, + {objectSDK.TypeLock, bucketNameLockers(idCnr, keys[1][:])}, } for i := range objs { @@ -244,7 +244,7 @@ func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object } } - return object.TypeRegular + return objectSDK.TypeRegular } // return true if provided object is of LOCK type. diff --git a/pkg/local_object_storage/metrics/blobovnizca.go b/pkg/local_object_storage/metrics/blobovnizca.go new file mode 100644 index 000000000..a498d822e --- /dev/null +++ b/pkg/local_object_storage/metrics/blobovnizca.go @@ -0,0 +1,98 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewBlobovniczaTreeMetrics(path string, m metrics_impl.BlobobvnizcaMetrics) blobovniczatree.Metrics { + return &blobovniczaTreeMetrics{ + path: path, + shardID: undefined, + m: m, + } +} + +type blobovniczaTreeMetrics struct { + shardID string + path string + m metrics_impl.BlobobvnizcaMetrics +} + +func (m *blobovniczaTreeMetrics) Blobovnizca() blobovnicza.Metrics { + return &blobovniczaMetrics{ + shardID: func() string { return m.shardID }, + path: m.path, + m: m.m, + } +} + +func (m *blobovniczaTreeMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *blobovniczaTreeMetrics) SetMode(readOnly bool) { + m.m.SetBlobobvnizcaTreeMode(m.shardID, m.path, readOnly) +} + +func (m *blobovniczaTreeMetrics) Close() { + m.m.CloseBlobobvnizcaTree(m.shardID, m.path) +} + +func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) +} + +func (m *blobovniczaTreeMetrics) Exists(d time.Duration, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Exists", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) +} + +func (m *blobovniczaTreeMetrics) GetRange(d time.Duration, size int, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "GetRange", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) + if success { + m.m.AddBlobobvnizcaTreeGet(m.shardID, m.path, size) + } +} + +func (m *blobovniczaTreeMetrics) Get(d time.Duration, size int, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Get", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) + if success { + m.m.AddBlobobvnizcaTreeGet(m.shardID, m.path, size) + } +} + +func (m *blobovniczaTreeMetrics) Iterate(d time.Duration, success bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Iterate", d, success, metrics_impl.NullBool{}) +} + +func (m *blobovniczaTreeMetrics) Put(d time.Duration, size int, success bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Put", d, success, metrics_impl.NullBool{}) + if success { + m.m.AddBlobobvnizcaTreePut(m.shardID, m.path, size) + } +} + +type blobovniczaMetrics struct { + m metrics_impl.BlobobvnizcaMetrics + shardID func() string + path string +} + +func (m *blobovniczaMetrics) AddSize(size uint64) { + m.m.AddTreeSize(m.shardID(), m.path, size) +} + +func (m *blobovniczaMetrics) SubSize(size uint64) { + m.m.SubTreeSize(m.shardID(), m.path, size) +} + +func (m *blobovniczaMetrics) IncOpenBlobovnizcaCount() { + m.m.IncOpenBlobovnizcaCount(m.shardID(), m.path) +} + +func (m *blobovniczaMetrics) DecOpenBlobovnizcaCount() { + m.m.DecOpenBlobovnizcaCount(m.shardID(), m.path) +} diff --git a/pkg/local_object_storage/metrics/blobstore.go b/pkg/local_object_storage/metrics/blobstore.go new file mode 100644 index 000000000..48249e89c --- /dev/null +++ b/pkg/local_object_storage/metrics/blobstore.go @@ -0,0 +1,65 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +type blobstoreMetrics struct { + shardID string + m metrics_impl.BlobstoreMetrics +} + +func NewBlobstoreMetrics(m metrics_impl.BlobstoreMetrics) blobstor.Metrics { + return &blobstoreMetrics{ + shardID: undefined, + m: m, + } +} + +func (m *blobstoreMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *blobstoreMetrics) SetMode(readOnly bool) { + m.m.SetMode(m.shardID, readOnly) +} + +func (m *blobstoreMetrics) Close() { + m.m.Close(m.shardID) +} + +func (m *blobstoreMetrics) Delete(d time.Duration, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Delete", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +} + +func (m *blobstoreMetrics) Exists(d time.Duration, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Exists", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +} + +func (m *blobstoreMetrics) GetRange(d time.Duration, size int, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "GetRange", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) + if success { + m.m.AddGet(m.shardID, size) + } +} + +func (m *blobstoreMetrics) Get(d time.Duration, size int, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Get", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) + if success { + m.m.AddGet(m.shardID, size) + } +} + +func (m *blobstoreMetrics) Iterate(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, "Iterate", d, success, metrics_impl.NullBool{}) +} + +func (m *blobstoreMetrics) Put(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, "Put", d, success, metrics_impl.NullBool{}) + if success { + m.m.AddPut(m.shardID, size) + } +} diff --git a/pkg/local_object_storage/metrics/consts.go b/pkg/local_object_storage/metrics/consts.go new file mode 100644 index 000000000..519930710 --- /dev/null +++ b/pkg/local_object_storage/metrics/consts.go @@ -0,0 +1,3 @@ +package metrics + +const undefined = "undefined" diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go new file mode 100644 index 000000000..d3749d9bc --- /dev/null +++ b/pkg/local_object_storage/metrics/fstree.go @@ -0,0 +1,61 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewFSTreeMetricsWithoutShardID(path string, m metrics_impl.FSTreeMetrics) fstree.Metrics { + return &fstreeMetrics{ + shardID: undefined, + path: path, + m: m, + } +} + +type fstreeMetrics struct { + shardID string + path string + m metrics_impl.FSTreeMetrics +} + +func (m *fstreeMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *fstreeMetrics) SetMode(readOnly bool) { + m.m.SetMode(m.shardID, m.path, readOnly) +} +func (m *fstreeMetrics) Close() { + m.m.Close(m.shardID, m.path) +} + +func (m *fstreeMetrics) Iterate(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Iterate", d, success) +} +func (m *fstreeMetrics) Delete(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Delete", d, success) +} +func (m *fstreeMetrics) Exists(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Exists", d, success) +} +func (m *fstreeMetrics) Put(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Put", d, success) + if success { + m.m.AddPut(m.shardID, m.path, size) + } +} +func (m *fstreeMetrics) Get(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Get", d, success) + if success { + m.m.AddGet(m.shardID, m.path, size) + } +} +func (m *fstreeMetrics) GetRange(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "GetRange", d, success) + if success { + m.m.AddGet(m.shardID, m.path, size) + } +} diff --git a/pkg/local_object_storage/metrics/metabase.go b/pkg/local_object_storage/metrics/metabase.go new file mode 100644 index 000000000..d0fb31936 --- /dev/null +++ b/pkg/local_object_storage/metrics/metabase.go @@ -0,0 +1,39 @@ +package metrics + +import ( + "time" + + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewMetabaseMetrics(path string, m metrics_impl.MetabaseMetrics) meta.Metrics { + return &metabaseMetrics{ + shardID: undefined, + path: path, + m: m, + } +} + +type metabaseMetrics struct { + shardID string + path string + m metrics_impl.MetabaseMetrics +} + +func (m *metabaseMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *metabaseMetrics) SetMode(mode mode.Mode) { + m.m.SetMode(m.shardID, m.path, mode.String()) +} + +func (m *metabaseMetrics) Close() { + m.m.Close(m.shardID, m.path) +} + +func (m *metabaseMetrics) AddMethodDuration(method string, d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, method, d, success) +} diff --git a/pkg/local_object_storage/metrics/pilorama.go b/pkg/local_object_storage/metrics/pilorama.go new file mode 100644 index 000000000..21f027a6e --- /dev/null +++ b/pkg/local_object_storage/metrics/pilorama.go @@ -0,0 +1,37 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewPiloramaMetrics(m metrics_impl.PiloramaMetrics) pilorama.Metrics { + return &piloramaMetrics{ + shardID: undefined, + m: m, + } +} + +type piloramaMetrics struct { + shardID string + m metrics_impl.PiloramaMetrics +} + +func (m *piloramaMetrics) SetParentID(id string) { + m.shardID = id +} + +func (m *piloramaMetrics) SetMode(mode mode.Mode) { + m.m.SetMode(m.shardID, mode) +} + +func (m *piloramaMetrics) Close() { + m.m.Close(m.shardID) +} + +func (m *piloramaMetrics) AddMethodDuration(method string, d time.Duration, success bool) { + m.m.AddMethodDuration(m.shardID, method, d, success) +} diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index 3065c8370..c65488b74 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -1,6 +1,7 @@ package pilorama import ( + "encoding/binary" "sort" "sync" "time" @@ -49,11 +50,80 @@ func (b *batch) run() { sort.Slice(b.operations, func(i, j int) bool { return b.operations[i].Time < b.operations[j].Time }) + b.operations = removeDuplicatesInPlace(b.operations) - var lm Move - return b.forest.applyOperation(bLog, bTree, b.operations, &lm) + // Our main use-case is addition of new items. In this case, + // we do not need to perform undo()/redo(), just do(). + // https://github.com/trvedata/move-op/blob/6c23447c12a7862ff31b7fc2205f6c90fbdb9dc0/proof/Move_Create.thy#L259 + // + // For this optimization to work we need to ensure three things: + // 1. The node itself is not yet in tree. + // 2. The node is not a parent. This case is not mentioned in the article, because + // they consider a "static order" (perform all CREATE operations before MOVE). + // We need this because if node _is_ a parent, we could violate (3) for some late operation. + // See TestForest_ApplySameOperation for details. + // 3. Parent of each operation is already in tree. + var parents map[uint64]struct{} + var cKey [maxKeySize]byte + var slow bool + for i := range b.operations { + _, _, _, inTree := b.forest.getState(bTree, stateKey(cKey[:], b.operations[i].Child)) + if inTree { + slow = true + break + } + + key := childrenKey(cKey[:], b.operations[i].Child, 0) + k, _ := bTree.Cursor().Seek(key) + if len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == b.operations[i].Child { + slow = true + break + } + + if b.operations[i].Parent == RootID { + continue + } else if parents == nil { + // Attaching key only to root is done frequently, + // no allocations are performed unless necessary. + parents = make(map[uint64]struct{}) + } else if _, ok := parents[b.operations[i].Parent]; ok { + continue + } + + p := b.operations[i].Parent + _, ts, _, inTree := b.forest.getState(bTree, stateKey(cKey[:], p)) + if !inTree || b.operations[0].Time < ts { + slow = true + break + } + parents[b.operations[i].Parent] = struct{}{} + } + + if slow { + var lm Move + return b.forest.applyOperation(bLog, bTree, b.operations, &lm) + } + + for i := range b.operations { + if err := b.forest.do(bLog, bTree, cKey[:], b.operations[i]); err != nil { + return err + } + } + return nil }) - for i := range b.operations { + for i := range b.results { b.results[i] <- err } } + +func removeDuplicatesInPlace(a []*Move) []*Move { + equalCount := 0 + for i := 1; i < len(a); i++ { + if a[i].Time == a[i-1].Time { + equalCount++ + } else { + a[i-equalCount] = a[i] + } + } + return a[:len(a)-equalCount] +} diff --git a/pkg/local_object_storage/pilorama/batch_test.go b/pkg/local_object_storage/pilorama/batch_test.go new file mode 100644 index 000000000..931fce18c --- /dev/null +++ b/pkg/local_object_storage/pilorama/batch_test.go @@ -0,0 +1,70 @@ +package pilorama + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_removeDuplicatesInPlace(t *testing.T) { + testCases := []struct { + before []int + after []int + }{ + { + before: []int{}, + after: []int{}, + }, + { + before: []int{1}, + after: []int{1}, + }, + { + before: []int{1, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 3}, + after: []int{1, 2, 3}, + }, + { + before: []int{1, 1, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 2, 3}, + after: []int{1, 2, 3}, + }, + { + before: []int{1, 1, 1}, + after: []int{1}, + }, + { + before: []int{1, 1, 2, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 1, 1, 2, 3, 3, 3}, + after: []int{1, 2, 3}, + }, + } + + for _, tc := range testCases { + ops := make([]*Move, len(tc.before)) + for i := range ops { + ops[i] = &Move{Meta: Meta{Time: Timestamp(tc.before[i])}} + } + + expected := make([]*Move, len(tc.after)) + for i := range expected { + expected[i] = &Move{Meta: Meta{Time: Timestamp(tc.after[i])}} + } + + actual := removeDuplicatesInPlace(ops) + require.Equal(t, expected, actual, "%d", tc.before) + } +} diff --git a/pkg/local_object_storage/pilorama/bench_test.go b/pkg/local_object_storage/pilorama/bench_test.go new file mode 100644 index 000000000..e729b9ea6 --- /dev/null +++ b/pkg/local_object_storage/pilorama/bench_test.go @@ -0,0 +1,55 @@ +package pilorama + +import ( + "context" + "os" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "github.com/stretchr/testify/require" +) + +func getTimestamp(reorder int, ts Timestamp) Timestamp { + base := ts / Timestamp(reorder) + rem := ts % Timestamp(reorder) + return base*Timestamp(reorder) + Timestamp(reorder) - rem +} + +func BenchmarkCreate(b *testing.B) { + // Use `os.TempDir` because we construct multiple times in the same test. + tmpDir, err := os.MkdirTemp(os.TempDir(), "*") + require.NoError(b, err) + + f := NewBoltForest( + WithPath(filepath.Join(tmpDir, "test.db")), + WithMaxBatchSize(runtime.GOMAXPROCS(0))) + require.NoError(b, f.Open(false)) + require.NoError(b, f.Init()) + b.Cleanup(func() { + require.NoError(b, f.Close()) + require.NoError(b, os.RemoveAll(tmpDir)) + }) + + cid := cidtest.ID() + treeID := "tree" + ctx := context.Background() + var index atomic.Int32 + index.Store(-1) + b.SetParallelism(2) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + i := index.Add(1) + op := &Move{ + Meta: Meta{Time: getTimestamp(runtime.GOMAXPROCS(0)*2, Timestamp(i+1))}, + Child: Node(i + 1), + Parent: RootID, + } + if err := f.TreeApply(ctx, cid, treeID, op, true); err != nil { + b.FailNow() + } + } + }) +} diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 1ecc89cb5..a729e2a22 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -12,10 +12,11 @@ import ( "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/io" "go.etcd.io/bbolt" @@ -36,6 +37,11 @@ type boltForest struct { cfg } +const ( + childrenKeySize = 17 + maxKeySize = childrenKeySize +) + var ( dataBucket = []byte{0} logBucket = []byte{1} @@ -71,6 +77,7 @@ func NewBoltForest(opts ...Option) ForestStorage { maxBatchDelay: bbolt.DefaultMaxBatchDelay, maxBatchSize: bbolt.DefaultMaxBatchSize, openFile: os.OpenFile, + metrics: &noopMetrics{}, }, } @@ -100,12 +107,13 @@ func (t *boltForest) SetMode(m mode.Mode) error { } t.mode = m + t.metrics.SetMode(m) return nil } func (t *boltForest) Open(readOnly bool) error { err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { - return fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err) + return metaerr.Wrap(fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)) } opts := *bbolt.DefaultOptions @@ -116,12 +124,16 @@ func (t *boltForest) Open(readOnly bool) error { t.db, err = bbolt.Open(t.path, t.perm, &opts) if err != nil { - return fmt.Errorf("can't open the pilorama DB: %w", err) + return metaerr.Wrap(fmt.Errorf("can't open the pilorama DB: %w", err)) } t.db.MaxBatchSize = t.maxBatchSize t.db.MaxBatchDelay = t.maxBatchDelay - + m := mode.ReadWrite + if readOnly { + m = mode.ReadOnly + } + t.metrics.SetMode(m) return nil } func (t *boltForest) Init() error { @@ -141,10 +153,18 @@ func (t *boltForest) Init() error { }) } func (t *boltForest) Close() error { + var err error if t.db != nil { - return t.db.Close() + err = t.db.Close() } - return nil + if err == nil { + t.metrics.Close() + } + return err +} + +func (t *boltForest) SetParentID(id string) { + t.metrics.SetParentID(id) } // TreeMove implements the Forest interface. @@ -174,7 +194,7 @@ func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID strin lm := *m fullID := bucketName(d.CID, treeID) - return &lm, t.db.Batch(func(tx *bbolt.Tx) error { + return &lm, metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { bLog, bTree, err := t.getTreeBuckets(tx, fullID) if err != nil { return err @@ -184,12 +204,54 @@ func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID strin if lm.Child == RootID { lm.Child = t.findSpareID(bTree) } - return t.do(bLog, bTree, make([]byte, 17), &lm) + return t.do(bLog, bTree, make([]byte, maxKeySize), &lm) + })) +} + +func (t *boltForest) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeHeight", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + t.modeMtx.RLock() + defer t.modeMtx.RUnlock() + + if t.mode.NoMetabase() { + return 0, ErrDegradedMode + } + + var height uint64 + var retErr error + err := t.db.View(func(tx *bbolt.Tx) error { + treeRoot := tx.Bucket(bucketName(cid, treeID)) + if treeRoot != nil { + k, _ := treeRoot.Bucket(logBucket).Cursor().Last() + height = binary.BigEndian.Uint64(k) + } else { + retErr = ErrTreeNotFound + } + return nil }) + if err == nil { + err = retErr + } + return height, metaerr.Wrap(err) } // TreeExists implements the Forest interface. func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeExists", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeExists", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -212,14 +274,22 @@ func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID strin exists = treeRoot != nil return nil }) - - return exists, err + success = err == nil + return exists, metaerr.Wrap(err) } var syncHeightKey = []byte{'h'} // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeUpdateLastSyncHeight", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeUpdateLastSyncHeight", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -233,7 +303,7 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID binary.LittleEndian.PutUint64(rawHeight, height) buck := bucketName(cid, treeID) - return t.db.Batch(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(buck) if treeRoot == nil { return ErrTreeNotFound @@ -241,11 +311,21 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID b := treeRoot.Bucket(dataBucket) return b.Put(syncHeightKey, rawHeight) - }) + })) + success = err == nil + return err } // TreeLastSyncHeight implements the pilorama.Forest interface. func (t *boltForest) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeLastSyncHeight", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeLastSyncHeight", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -270,11 +350,20 @@ func (t *boltForest) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, tree } return nil }) - return height, err + success = err == nil + return height, metaerr.Wrap(err) } // TreeAddByPath implements the Forest interface. func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID string, attr string, path []string, meta []KeyValue) ([]Move, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeAddByPath", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeAddByPath", trace.WithAttributes( attribute.String("container_id", d.CID.EncodeToString()), @@ -288,6 +377,12 @@ func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID ) defer span.End() + res, err := t.addByPathInternal(d, attr, treeID, path, meta) + success = err == nil + return res, err +} + +func (t *boltForest) addByPathInternal(d CIDDescriptor, attr string, treeID string, path []string, meta []KeyValue) ([]Move, error) { if !d.checkValid() { return nil, ErrInvalidCIDDescriptor } @@ -305,7 +400,7 @@ func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID } var lm []Move - var key [17]byte + var key [maxKeySize]byte fullID := bucketName(d.CID, treeID) err := t.db.Batch(func(tx *bbolt.Tx) error { @@ -350,7 +445,7 @@ func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID } return t.do(bLog, bTree, key[:], &lm[len(lm)-1]) }) - return lm, err + return lm, metaerr.Wrap(err) } // getLatestTimestamp returns timestamp for a new operation which is guaranteed to be bigger than @@ -382,6 +477,14 @@ func (t *boltForest) findSpareID(bTree *bbolt.Bucket) uint64 { // TreeApply implements the Forest interface. func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string, m *Move, backgroundSync bool) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeApply", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeApply", trace.WithAttributes( attribute.String("container_id", cnr.EncodeToString()), @@ -405,6 +508,7 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string err := t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cnr, treeID)) if treeRoot == nil { + success = true return nil } @@ -413,16 +517,18 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string var logKey [8]byte binary.BigEndian.PutUint64(logKey[:], m.Time) seen = b.Get(logKey[:]) != nil + success = true return nil }) if err != nil || seen { - return err + success = err == nil + return metaerr.Wrap(err) } } if t.db.MaxBatchSize == 1 { fullID := bucketName(cnr, treeID) - return t.db.Update(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.Update(func(tx *bbolt.Tx) error { bLog, bTree, err := t.getTreeBuckets(tx, fullID) if err != nil { return err @@ -430,12 +536,16 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string var lm Move return t.applyOperation(bLog, bTree, []*Move{m}, &lm) - }) + })) + success = err == nil + return err } ch := make(chan error, 1) t.addBatch(cnr, treeID, m, ch) - return <-ch + err := <-ch + success = err == nil + return metaerr.Wrap(err) } func (t *boltForest) addBatch(cnr cidSDK.ID, treeID string, m *Move, ch chan error) { @@ -507,7 +617,7 @@ func (t *boltForest) getTreeBuckets(tx *bbolt.Tx, treeRoot []byte) (*bbolt.Bucke // applyOperations applies log operations. Assumes lm are sorted by timestamp. func (t *boltForest) applyOperation(logBucket, treeBucket *bbolt.Bucket, ms []*Move, lm *Move) error { var tmp Move - var cKey [17]byte + var cKey [maxKeySize]byte c := logBucket.Cursor() @@ -689,6 +799,14 @@ func (t *boltForest) isAncestor(b *bbolt.Bucket, parent, child Node) bool { // TreeGetByPath implements the Forest interface. func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetByPath", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetByPath", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -705,6 +823,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st } if len(path) == 0 { + success = true return nil, nil } @@ -717,7 +836,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st var nodes []Node - return nodes, t.db.View(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cid, treeID)) if treeRoot == nil { return ErrTreeNotFound @@ -754,11 +873,21 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st childKey, _ = c.Next() } return nil - }) + })) + success = err == nil + return nodes, err } // TreeGetMeta implements the forest interface. func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetMeta", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetMeta", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -793,12 +922,20 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri _, _, meta, _ := t.getState(b, stateKey(key, nodeID)) return m.FromBytes(meta) }) - - return m, parentID, err + success = err == nil + return m, parentID, metaerr.Wrap(err) } // TreeGetChildren implements the Forest interface. -func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { +func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetChildren", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetChildren", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -819,7 +956,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID key[0] = 'c' binary.LittleEndian.PutUint64(key[1:], nodeID) - var children []uint64 + var result []NodeInfo err := t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cid, treeID)) @@ -829,17 +966,36 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID b := treeRoot.Bucket(dataBucket) c := b.Cursor() - for k, _ := c.Seek(key); len(k) == 17 && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { - children = append(children, binary.LittleEndian.Uint64(k[9:])) + for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { + childID := binary.LittleEndian.Uint64(k[9:]) + childInfo := NodeInfo{ + ID: childID, + } + parentID, _, metaBytes, found := t.getState(b, stateKey(key, childID)) + if found { + childInfo.ParentID = parentID + if err := childInfo.Meta.FromBytes(metaBytes); err != nil { + return err + } + } + result = append(result, childInfo) } return nil }) - - return children, err + success = err == nil + return result, metaerr.Wrap(err) } // TreeList implements the Forest interface. func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeList", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeList", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -873,14 +1029,22 @@ func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, err return nil }) if err != nil { - return nil, fmt.Errorf("could not list trees: %w", err) + return nil, metaerr.Wrap(fmt.Errorf("could not list trees: %w", err)) } - + success = true return ids, nil } // TreeGetOpLog implements the pilorama.Forest interface. func (t *boltForest) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetOpLog", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetOpLog", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -914,12 +1078,20 @@ func (t *boltForest) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID str } return nil }) - - return lm, err + success = err == nil + return lm, metaerr.Wrap(err) } // TreeDrop implements the pilorama.Forest interface. func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeDrop", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeDrop", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -937,7 +1109,7 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) return ErrReadOnlyMode } - return t.db.Batch(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { if treeID == "" { c := tx.Cursor() prefix := make([]byte, 32) @@ -955,7 +1127,9 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) return ErrTreeNotFound } return err - }) + })) + success = err == nil + return err } func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) { @@ -1058,7 +1232,7 @@ func childrenKey(key []byte, child, parent Node) []byte { key[0] = 'c' binary.LittleEndian.PutUint64(key[1:], parent) binary.LittleEndian.PutUint64(key[9:], child) - return key[:17] + return key[:childrenKeySize] } // 'i' + attribute name (string) + attribute value (string) + parent (id) + node (id) -> 0/1. diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 672a38edd..8fb519128 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -119,6 +119,7 @@ func (f *memoryForest) SetMode(mode.Mode) error { func (f *memoryForest) Close() error { return nil } +func (f *memoryForest) SetParentID(string) {} // TreeGetByPath implements the Forest interface. func (f *memoryForest) TreeGetByPath(_ context.Context, cid cid.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { @@ -147,7 +148,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, } // TreeGetChildren implements the Forest interface. -func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]uint64, error) { +func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]NodeInfo, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -155,8 +156,14 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str } children := s.tree.getChildren(nodeID) - res := make([]Node, len(children)) - copy(res, children) + res := make([]NodeInfo, 0, len(children)) + for _, childID := range children { + res = append(res, NodeInfo{ + ID: childID, + Meta: s.infoMap[childID].Meta, + ParentID: s.infoMap[childID].Parent, + }) + } return res, nil } @@ -214,6 +221,15 @@ func (f *memoryForest) TreeList(_ context.Context, cid cid.ID) ([]string, error) return res, nil } +func (f *memoryForest) TreeHeight(_ context.Context, cid cid.ID, treeID string) (uint64, error) { + fullID := cid.EncodeToString() + "/" + treeID + tree, ok := f.treeMap[fullID] + if !ok { + return 0, ErrTreeNotFound + } + return tree.operations[len(tree.operations)-1].Time, nil +} + // TreeExists implements the pilorama.Forest interface. func (f *memoryForest) TreeExists(_ context.Context, cid cid.ID, treeID string) (bool, error) { fullID := cid.EncodeToString() + "/" + treeID diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 0cff28c3f..5c143d3eb 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -13,6 +13,7 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) var providers = []struct { @@ -146,16 +147,23 @@ func testForestTreeGetChildren(t *testing.T, s Forest) { treeAdd(t, 2, 0) treeAdd(t, 7, 0) - testGetChildren := func(t *testing.T, nodeID Node, expected []Node) { + testGetChildren := func(t *testing.T, nodeID Node, expected []NodeInfo) { actual, err := s.TreeGetChildren(context.Background(), cid, treeID, nodeID) require.NoError(t, err) require.ElementsMatch(t, expected, actual) } - testGetChildren(t, 0, []uint64{10, 2, 7}) - testGetChildren(t, 10, []uint64{3, 6}) + testGetChildren(t, 0, []NodeInfo{ + {ID: 10, Meta: Meta{Time: 1, Items: []KeyValue{}}}, + {ID: 2, Meta: Meta{Time: 5, Items: []KeyValue{}}}, + {ID: 7, Meta: Meta{Time: 6, Items: []KeyValue{}}}, + }) + testGetChildren(t, 10, []NodeInfo{ + {ID: 3, ParentID: 10, Meta: Meta{Time: 2, Items: []KeyValue{}}}, + {ID: 6, ParentID: 10, Meta: Meta{Time: 3, Items: []KeyValue{}}}, + }) testGetChildren(t, 3, nil) - testGetChildren(t, 6, []uint64{11}) + testGetChildren(t, 6, []NodeInfo{{ID: 11, ParentID: 6, Meta: Meta{Time: 4, Items: []KeyValue{}}}}) testGetChildren(t, 11, nil) testGetChildren(t, 2, nil) testGetChildren(t, 7, nil) @@ -444,6 +452,83 @@ func testForestTreeApply(t *testing.T, constructor func(t testing.TB, _ ...Optio }) } +func TestForest_ApplySameOperation(t *testing.T) { + for i := range providers { + t.Run(providers[i].name, func(t *testing.T) { + parallel := providers[i].name != "inmemory" + testForestApplySameOperation(t, providers[i].construct, parallel) + }) + } +} + +func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ ...Option) Forest, parallel bool) { + cid := cidtest.ID() + treeID := "version" + + batchSize := 3 + ctx := context.Background() + errG, _ := errgroup.WithContext(ctx) + if !parallel { + batchSize = 1 + errG.SetLimit(1) + } + + meta := []Meta{ + {Time: 1, Items: []KeyValue{{AttributeFilename, []byte("1")}, {"attr", []byte{1}}}}, + {Time: 2, Items: []KeyValue{{AttributeFilename, []byte("2")}, {"attr", []byte{1}}}}, + {Time: 3, Items: []KeyValue{{AttributeFilename, []byte("3")}, {"attr", []byte{1}}}}, + } + logs := []Move{ + { + Child: 1, + Parent: RootID, + Meta: meta[0], + }, + { + Child: 2, + Parent: 1, + Meta: meta[1], + }, + { + Child: 1, + Parent: 2, + Meta: meta[2], + }, + } + + check := func(t *testing.T, s Forest) { + testMeta(t, s, cid, treeID, 1, RootID, meta[0]) + testMeta(t, s, cid, treeID, 2, 1, meta[1]) + + nodes, err := s.TreeGetChildren(ctx, cid, treeID, RootID) + require.NoError(t, err) + require.Equal(t, []NodeInfo{{ID: 1, ParentID: RootID, Meta: meta[0]}}, nodes) + + nodes, err = s.TreeGetChildren(ctx, cid, treeID, 1) + require.NoError(t, err) + require.Equal(t, []NodeInfo{{ID: 2, ParentID: 1, Meta: meta[1]}}, nodes) + } + + t.Run("expected", func(t *testing.T) { + s := constructor(t) + for i := range logs { + require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[i], false)) + } + check(t, s) + }) + + s := constructor(t, WithMaxBatchSize(batchSize)) + require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[0], false)) + for i := 0; i < batchSize; i++ { + errG.Go(func() error { + return s.TreeApply(ctx, cid, treeID, &logs[2], false) + }) + } + require.NoError(t, errG.Wait()) + require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[1], false)) + check(t, s) +} + func TestForest_GetOpLog(t *testing.T) { for i := range providers { t.Run(providers[i].name, func(t *testing.T) { @@ -526,10 +611,19 @@ func testForestTreeExists(t *testing.T, constructor func(t testing.TB, opts ...O checkExists(t, false, cid, treeID) }) - require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &Move{Parent: 0, Child: 1}, false)) + require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &Move{Meta: Meta{Time: 11}, Parent: 0, Child: 1}, false)) checkExists(t, true, cid, treeID) + + height, err := s.TreeHeight(context.Background(), cid, treeID) + require.NoError(t, err) + require.EqualValues(t, 11, height) + checkExists(t, false, cidtest.ID(), treeID) // different CID, same tree - checkExists(t, false, cid, "another tree") // same CID, different tree + + _, err = s.TreeHeight(context.Background(), cidtest.ID(), treeID) + require.ErrorIs(t, err, ErrTreeNotFound) + + checkExists(t, false, cid, "another tree") // same CID, different tree t.Run("can be removed", func(t *testing.T) { require.NoError(t, s.TreeDrop(context.Background(), cid, treeID)) diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index 1bde312ac..c9f5df3b7 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -68,10 +68,14 @@ func (s *memoryTree) Apply(op *Move) error { // do performs a single move operation on a tree. func (s *memoryTree) do(op *Move) move { + m := op.Meta + if m.Items == nil { + m.Items = []KeyValue{} + } lm := move{ Move: Move{ Parent: op.Parent, - Meta: op.Meta, + Meta: m, Child: op.Child, }, } @@ -91,7 +95,7 @@ func (s *memoryTree) do(op *Move) move { p.Meta.Time = op.Time } - p.Meta = op.Meta + p.Meta = m p.Parent = op.Parent s.tree.infoMap[op.Child] = p diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 9ca721be8..ea171a479 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -32,7 +32,7 @@ type Forest interface { TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) // TreeGetChildren returns children of the node with the specified ID. The order is arbitrary. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) + TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) // TreeGetOpLog returns first log operation stored at or above the height. // In case no such operation is found, empty Move and nil error should be returned. TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) @@ -50,6 +50,8 @@ type Forest interface { TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error // TreeLastSyncHeight returns last log height synchronized with _all_ container nodes. TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) + // TreeHeight returns current tree height. + TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) } type ForestStorage interface { @@ -59,6 +61,7 @@ type ForestStorage interface { Open(bool) error Close() error SetMode(m mode.Mode) error + SetParentID(id string) Forest } diff --git a/pkg/local_object_storage/pilorama/metrics.go b/pkg/local_object_storage/pilorama/metrics.go new file mode 100644 index 000000000..543ad3e31 --- /dev/null +++ b/pkg/local_object_storage/pilorama/metrics.go @@ -0,0 +1,23 @@ +package pilorama + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) + +type Metrics interface { + SetParentID(id string) + + SetMode(m mode.Mode) + Close() + + AddMethodDuration(method string, d time.Duration, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetParentID(string) {} +func (m *noopMetrics) SetMode(mode.Mode) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/pilorama/option.go b/pkg/local_object_storage/pilorama/option.go index 0dd5e63d4..d576d427f 100644 --- a/pkg/local_object_storage/pilorama/option.go +++ b/pkg/local_object_storage/pilorama/option.go @@ -15,6 +15,7 @@ type cfg struct { maxBatchDelay time.Duration maxBatchSize int openFile func(string, int, fs.FileMode) (*os.File, error) + metrics Metrics } func WithPath(path string) Option { @@ -52,3 +53,9 @@ func WithOpenFile(openFile func(string, int, fs.FileMode) (*os.File, error)) Opt c.openFile = openFile } } + +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/pilorama/types.go b/pkg/local_object_storage/pilorama/types.go index 99918683d..8d8616364 100644 --- a/pkg/local_object_storage/pilorama/types.go +++ b/pkg/local_object_storage/pilorama/types.go @@ -55,3 +55,9 @@ var ( func isAttributeInternal(key string) bool { return key == AttributeFilename } + +type NodeInfo struct { + ID Node + Meta Meta + ParentID Node +} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index e74f235f8..db8248c02 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -5,12 +5,12 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" @@ -98,7 +98,7 @@ func (s *Shard) Init(ctx context.Context) error { if !s.GetMode().NoMetabase() { var initMetabase initializer - if s.needRefillMetabase() { + if s.NeedRefillMetabase() { initMetabase = (*metabaseSynchronizer)(s) } else { initMetabase = s.metaBase @@ -138,7 +138,7 @@ func (s *Shard) Init(ctx context.Context) error { } } - s.updateMetrics() + s.updateMetrics(ctx) s.gc = &gc{ gcCfg: &s.gcCfg, @@ -170,7 +170,7 @@ func (s *Shard) refillMetabase(ctx context.Context) error { obj := objectSDK.New() - err = blobstor.IterateBinaryObjects(s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { + err = blobstor.IterateBinaryObjects(ctx, s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { if err := obj.Unmarshal(data); err != nil { s.log.Warn(logs.ShardCouldNotUnmarshalObject, zap.Stringer("address", addr), @@ -304,8 +304,8 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { opts[i](&c) } - s.m.Lock() - defer s.m.Unlock() + unlock := s.lockExclusive() + defer unlock() ok, err := s.metaBase.Reload(c.metaOpts...) if err != nil { @@ -335,3 +335,15 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { s.log.Info(logs.ShardTryingToRestoreReadwriteMode) return s.setMode(mode.ReadWrite) } + +func (s *Shard) lockExclusive() func() { + s.setModeRequested.Store(true) + val := s.gcCancel.Load() + if val != nil { + cancelGC := val.(context.CancelFunc) + cancelGC() + } + s.m.Lock() + s.setModeRequested.Store(false) + return s.m.Unlock +} diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 170052d63..82b107196 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -6,6 +6,7 @@ import ( "math" "os" "path/filepath" + "sync/atomic" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -27,7 +28,6 @@ import ( objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap/zaptest" ) @@ -43,6 +43,8 @@ type objAddr struct { } func TestShardOpen(t *testing.T) { + t.Parallel() + dir := t.TempDir() metaPath := filepath.Join(dir, "meta") @@ -111,6 +113,8 @@ func TestShardOpen(t *testing.T) { } func TestRefillMetabaseCorrupted(t *testing.T) { + t.Parallel() + dir := t.TempDir() fsTree := fstree.New( @@ -164,6 +168,8 @@ func TestRefillMetabaseCorrupted(t *testing.T) { } func TestRefillMetabase(t *testing.T) { + t.Parallel() + p := t.Name() defer os.RemoveAll(p) diff --git a/pkg/local_object_storage/shard/count.go b/pkg/local_object_storage/shard/count.go new file mode 100644 index 000000000..abed5278e --- /dev/null +++ b/pkg/local_object_storage/shard/count.go @@ -0,0 +1,31 @@ +package shard + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// LogicalObjectsCount returns logical objects count. +func (s *Shard) LogicalObjectsCount(ctx context.Context) (uint64, error) { + _, span := tracing.StartSpanFromContext(ctx, "Shard.LogicalObjectsCount", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.GetMode().NoMetabase() { + return 0, ErrDegradedMode + } + + cc, err := s.metaBase.ObjectCounters() + if err != nil { + return 0, err + } + return cc.Logic(), nil +} diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index f086aa30f..2c7e0af27 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -4,11 +4,11 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -21,7 +21,9 @@ type DeletePrm struct { } // DeleteRes groups the resulting values of Delete operation. -type DeleteRes struct{} +type DeleteRes struct { + deleted uint64 +} // SetAddresses is a Delete option to set the addresses of the objects to delete. // @@ -53,70 +55,76 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { return DeleteRes{}, ErrDegradedMode } - ln := len(prm.addr) - - smalls := make(map[oid.Address][]byte, ln) - - for i := range prm.addr { - if s.hasWriteCache() { - err := s.writeCache.Delete(ctx, prm.addr[i]) - if err != nil && !IsErrNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { - s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.String("error", err.Error())) - } + result := DeleteRes{} + for _, addr := range prm.addr { + select { + case <-ctx.Done(): + return result, ctx.Err() + default: } - var sPrm meta.StorageIDPrm - sPrm.SetAddress(prm.addr[i]) + s.deleteObjectFromWriteCacheSafe(ctx, addr) - res, err := s.metaBase.StorageID(ctx, sPrm) - if err != nil { - s.log.Debug(logs.ShardCantGetStorageIDFromMetabase, - zap.Stringer("object", prm.addr[i]), - zap.String("error", err.Error())) + s.deleteFromBlobstorSafe(ctx, addr) - continue - } - - if res.StorageID() != nil { - smalls[prm.addr[i]] = res.StorageID() + if err := s.deleteFromMetabase(ctx, addr); err != nil { + return result, err // stop on metabase error ? } + result.deleted++ } + return result, nil +} + +func (s *Shard) deleteObjectFromWriteCacheSafe(ctx context.Context, addr oid.Address) { + if s.hasWriteCache() { + err := s.writeCache.Delete(ctx, addr) + if err != nil && !IsErrNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { + s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.Error(err)) + } + } +} + +func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { + var sPrm meta.StorageIDPrm + sPrm.SetAddress(addr) + + res, err := s.metaBase.StorageID(ctx, sPrm) + if err != nil { + s.log.Debug("can't get storage ID from metabase", + zap.Stringer("object", addr), + zap.String("error", err.Error())) + } + storageID := res.StorageID() + + var delPrm common.DeletePrm + delPrm.Address = addr + delPrm.StorageID = storageID + + _, err = s.blobStor.Delete(ctx, delPrm) + if err != nil { + s.log.Debug("can't remove object from blobStor", + zap.Stringer("object_address", addr), + zap.String("error", err.Error())) + } +} + +func (s *Shard) deleteFromMetabase(ctx context.Context, addr oid.Address) error { var delPrm meta.DeletePrm - delPrm.SetAddresses(prm.addr...) + delPrm.SetAddresses(addr) res, err := s.metaBase.Delete(ctx, delPrm) if err != nil { - return DeleteRes{}, err // stop on metabase error ? + return err } - - var totalRemovedPayload uint64 - s.decObjectCounterBy(physical, res.RawObjectsRemoved()) s.decObjectCounterBy(logical, res.AvailableObjectsRemoved()) - for i := range prm.addr { - removedPayload := res.RemovedPhysicalObjectSizes()[i] - totalRemovedPayload += removedPayload - logicalRemovedPayload := res.RemovedLogicalObjectSizes()[i] - if logicalRemovedPayload > 0 { - s.addToContainerSize(prm.addr[i].Container().EncodeToString(), -int64(logicalRemovedPayload)) - } + removedPayload := res.RemovedPhysicalObjectSizes()[0] + logicalRemovedPayload := res.RemovedLogicalObjectSizes()[0] + if logicalRemovedPayload > 0 { + s.addToContainerSize(addr.Container().EncodeToString(), -int64(logicalRemovedPayload)) } - s.addToPayloadSize(-int64(totalRemovedPayload)) + s.addToPayloadSize(-int64(removedPayload)) - for i := range prm.addr { - var delPrm common.DeletePrm - delPrm.Address = prm.addr[i] - id := smalls[prm.addr[i]] - delPrm.StorageID = id - - _, err = s.blobStor.Delete(ctx, delPrm) - if err != nil { - s.log.Debug(logs.ShardCantRemoveObjectFromBlobStor, - zap.Stringer("object_address", prm.addr[i]), - zap.String("error", err.Error())) - } - } - - return DeleteRes{}, nil + return nil } diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 9646e9aa0..441e1c455 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -13,11 +13,15 @@ import ( ) func TestShard_Delete(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardDelete(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardDelete(t, true) }) } diff --git a/pkg/local_object_storage/shard/errors.go b/pkg/local_object_storage/shard/errors.go index 3e5224eb9..2958a492c 100644 --- a/pkg/local_object_storage/shard/errors.go +++ b/pkg/local_object_storage/shard/errors.go @@ -4,9 +4,12 @@ import ( "errors" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) +var ErrShardDisabled = logicerr.New("shard disabled") + // IsErrNotFound checks if error returned by Shard Get/Head/GetRange method // corresponds to missing object. func IsErrNotFound(err error) bool { diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 66c61fccc..2cdb8dfa8 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -3,9 +3,9 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -38,6 +38,7 @@ func (p ExistsRes) Exists() bool { // // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Exists", trace.WithAttributes( @@ -52,7 +53,9 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { s.m.RLock() defer s.m.RUnlock() - if s.info.Mode.NoMetabase() { + if s.info.Mode.Disabled() { + return ExistsRes{}, ErrShardDisabled + } else if s.info.Mode.NoMetabase() { var p common.ExistsPrm p.Address = prm.addr diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 86995cd06..2221d57c1 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -67,6 +67,32 @@ type eventHandlers struct { handlers []eventHandler } +type gcRunResult struct { + success bool + deleted uint64 + failedToDelete uint64 +} + +const ( + objectTypeLock = "lock" + objectTypeTombstone = "tombstone" + objectTypeRegular = "regular" +) + +type GCMectrics interface { + AddRunDuration(d time.Duration, success bool) + AddDeletedCount(deleted, failed uint64) + AddExpiredObjectCollectionDuration(d time.Duration, success bool, objectType string) + AddInhumedObjectCount(count uint64, objectType string) +} + +type noopGCMetrics struct{} + +func (m *noopGCMetrics) AddRunDuration(time.Duration, bool) {} +func (m *noopGCMetrics) AddDeletedCount(uint64, uint64) {} +func (m *noopGCMetrics) AddExpiredObjectCollectionDuration(time.Duration, bool, string) {} +func (m *noopGCMetrics) AddInhumedObjectCount(uint64, string) {} + type gc struct { *gcCfg @@ -76,7 +102,7 @@ type gc struct { workerPool util.WorkerPool - remover func() + remover func(context.Context) gcRunResult eventChan chan Event mEventHandler map[eventType]*eventHandlers @@ -91,6 +117,8 @@ type gcCfg struct { expiredCollectorWorkersCount int expiredCollectorBatchSize int + + metrics GCMectrics } func defaultGCCfg() gcCfg { @@ -100,6 +128,7 @@ func defaultGCCfg() gcCfg { workerPoolInit: func(int) util.WorkerPool { return nil }, + metrics: &noopGCMetrics{}, } } @@ -115,7 +144,7 @@ func (gc *gc) init(ctx context.Context) { } gc.wg.Add(2) - go gc.tickRemover() + go gc.tickRemover(ctx) go gc.listenEvents(ctx) } @@ -146,8 +175,8 @@ func (gc *gc) listenEvents(ctx context.Context) { h := v.handlers[i] err := gc.workerPool.Submit(func() { + defer v.prevGroup.Done() h(runCtx, event) - v.prevGroup.Done() }) if err != nil { gc.log.Warn(logs.ShardCouldNotSubmitGCJobToWorkerPool, @@ -160,7 +189,7 @@ func (gc *gc) listenEvents(ctx context.Context) { } } -func (gc *gc) tickRemover() { +func (gc *gc) tickRemover(ctx context.Context) { defer gc.wg.Done() timer := time.NewTimer(gc.removerInterval) @@ -178,8 +207,13 @@ func (gc *gc) tickRemover() { gc.log.Debug(logs.ShardGCIsStopped) return case <-timer.C: - gc.remover() + startedAt := time.Now() + + result := gc.remover(ctx) timer.Reset(gc.removerInterval) + + gc.metrics.AddRunDuration(time.Since(startedAt), result.success) + gc.metrics.AddDeletedCount(result.deleted, result.failedToDelete) } } } @@ -196,7 +230,15 @@ func (gc *gc) stop() { // iterates over metabase and deletes objects // with GC-marked graves. // Does nothing if shard is in "read-only" mode. -func (s *Shard) removeGarbage() { +func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { + ctx, cancel := context.WithCancel(pctx) + defer cancel() + + s.gcCancel.Store(cancel) + if s.setModeRequested.Load() { + return + } + s.m.RLock() defer s.m.RUnlock() @@ -204,10 +246,19 @@ func (s *Shard) removeGarbage() { return } + s.log.Debug(logs.ShardGCRemoveGarbageStarted) + defer s.log.Debug(logs.ShardGCRemoveGarbageCompleted) + buf := make([]oid.Address, 0, s.rmBatchSize) var iterPrm meta.GarbageIterationPrm iterPrm.SetHandler(func(g meta.GarbageObject) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + buf = append(buf, g.Address()) if len(buf) == s.rmBatchSize { @@ -219,7 +270,7 @@ func (s *Shard) removeGarbage() { // iterate over metabase's objects with GC mark // (no more than s.rmBatchSize objects) - err := s.metaBase.IterateOverGarbage(iterPrm) + err := s.metaBase.IterateOverGarbage(ctx, iterPrm) if err != nil { s.log.Warn(logs.ShardIteratorOverMetabaseGraveyardFailed, zap.String("error", err.Error()), @@ -227,6 +278,7 @@ func (s *Shard) removeGarbage() { return } else if len(buf) == 0 { + result.success = true return } @@ -234,14 +286,20 @@ func (s *Shard) removeGarbage() { deletePrm.SetAddresses(buf...) // delete accumulated objects - _, err = s.delete(context.TODO(), deletePrm) + res, err := s.delete(ctx, deletePrm) + + result.deleted = res.deleted + result.failedToDelete = uint64(len(buf)) - res.deleted + result.success = true + if err != nil { s.log.Warn(logs.ShardCouldNotDeleteTheObjects, zap.String("error", err.Error()), ) - - return + result.success = false } + + return } func (s *Shard) getExpiredObjectsParameters() (workersCount, batchSize int) { @@ -259,6 +317,16 @@ func (s *Shard) getExpiredObjectsParameters() (workersCount, batchSize int) { } func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { + var err error + startedAt := time.Now() + + defer func() { + s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeRegular) + }() + + s.log.Debug(logs.ShardGCCollectingExpiredObjectsStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) + defer s.log.Debug(logs.ShardGCCollectingExpiredObjectsCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) + workersCount, batchSize := s.getExpiredObjectsParameters() errGroup, egCtx := errgroup.WithContext(ctx) @@ -266,8 +334,8 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { errGroup.Go(func() error { batch := make([]oid.Address, 0, batchSize) - err := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { - if o.Type() != object.TypeTombstone && o.Type() != object.TypeLock { + expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { + if o.Type() != objectSDK.TypeTombstone && o.Type() != objectSDK.TypeLock { batch = append(batch, o.Address()) if len(batch) == batchSize { @@ -280,8 +348,8 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { } } }) - if err != nil { - return err + if expErr != nil { + return expErr } if len(batch) > 0 { @@ -295,7 +363,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { return nil }) - if err := errGroup.Wait(); err != nil { + if err = errGroup.Wait(); err != nil { s.log.Warn(logs.ShardIteratorOverExpiredObjectsFailed, zap.String("error", err.Error())) } } @@ -314,6 +382,12 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } + expired, err := s.getExpiredWithLinked(ctx, expired) + if err != nil { + s.log.Warn(logs.ShardGCFailedToGetExpiredWithLinked, zap.Error(err)) + return + } + var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(expired...) @@ -329,6 +403,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } + s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeRegular) s.decObjectCounterBy(logical, res.AvailableInhumed()) i := 0 @@ -339,11 +414,33 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) } } +func (s *Shard) getExpiredWithLinked(ctx context.Context, source []oid.Address) ([]oid.Address, error) { + result := make([]oid.Address, 0, len(source)) + parentToChildren, err := s.metaBase.GetChildren(ctx, source) + if err != nil { + return nil, err + } + for parent, children := range parentToChildren { + result = append(result, parent) + result = append(result, children...) + } + + return result, nil +} + func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { + var err error + startedAt := time.Now() + + defer func() { + s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeTombstone) + }() + epoch := e.(newEpoch).epoch log := s.log.With(zap.Uint64("epoch", epoch)) log.Debug(logs.ShardStartedExpiredTombstonesHandling) + defer log.Debug(logs.ShardFinishedExpiredTombstonesHandling) const tssDeleteBatch = 50 tss := make([]meta.TombstonedObject, 0, tssDeleteBatch) @@ -372,7 +469,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { return } - err := s.metaBase.IterateOverGraveyard(iterPrm) + err = s.metaBase.IterateOverGraveyard(ctx, iterPrm) if err != nil { log.Error(logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) s.m.RUnlock() @@ -400,11 +497,19 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { tss = tss[:0] tssExp = tssExp[:0] } - - log.Debug(logs.ShardFinishedExpiredTombstonesHandling) } func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { + var err error + startedAt := time.Now() + + defer func() { + s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeLock) + }() + + s.log.Debug(logs.ShardGCCollectingExpiredLocksStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) + defer s.log.Debug(logs.ShardGCCollectingExpiredLocksCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) + workersCount, batchSize := s.getExpiredObjectsParameters() errGroup, egCtx := errgroup.WithContext(ctx) @@ -413,8 +518,8 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { errGroup.Go(func() error { batch := make([]oid.Address, 0, batchSize) - err := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { - if o.Type() == object.TypeLock { + expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { + if o.Type() == objectSDK.TypeLock { batch = append(batch, o.Address()) if len(batch) == batchSize { @@ -427,8 +532,8 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { } } }) - if err != nil { - return err + if expErr != nil { + return expErr } if len(batch) > 0 { @@ -442,7 +547,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { return nil }) - if err := errGroup.Wait(); err != nil { + if err = errGroup.Wait(); err != nil { s.log.Warn(logs.ShardIteratorOverExpiredLocksFailed, zap.String("error", err.Error())) } } @@ -455,7 +560,7 @@ func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, onExpiredFo return ErrDegradedMode } - err := s.metaBase.IterateExpired(epoch, func(expiredObject *meta.ExpiredObject) error { + err := s.metaBase.IterateExpired(ctx, epoch, func(expiredObject *meta.ExpiredObject) error { select { case <-ctx.Done(): return meta.ErrInterruptIterator @@ -511,6 +616,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston return } + s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeTombstone) s.decObjectCounterBy(logical, res.AvailableInhumed()) i := 0 @@ -522,7 +628,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston // drop just processed expired tombstones // from graveyard - err = s.metaBase.DropGraves(tss) + err = s.metaBase.DropGraves(ctx, tss) if err != nil { s.log.Warn(logs.ShardCouldNotDropExpiredGraveRecords, zap.Error(err)) } @@ -556,6 +662,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] return } + s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeLock) s.decObjectCounterBy(logical, res.AvailableInhumed()) i := 0 diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go new file mode 100644 index 000000000..bc895d67b --- /dev/null +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -0,0 +1,144 @@ +package shard + +import ( + "context" + "path/filepath" + "testing" + "time" + + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/panjf2000/ants/v2" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" +) + +func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { + t.Parallel() + + rootPath := t.TempDir() + + var sh *Shard + + l := &logger.Logger{Logger: zaptest.NewLogger(t)} + blobOpts := []blobstor.Option{ + blobstor.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(1), + blobovniczatree.WithBlobovniczaShallowWidth(1)), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return len(data) <= 1<<20 + }, + }, + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(rootPath, "blob"))), + }, + }), + } + + opts := []Option{ + WithID(NewIDFromBytes([]byte{})), + WithLogger(l), + WithBlobStorOptions(blobOpts...), + WithMetaBaseOptions( + meta.WithPath(filepath.Join(rootPath, "meta")), + meta.WithEpochState(epochState{}), + ), + WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), + WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(addresses) + }), + WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { + sh.HandleExpiredLocks(ctx, epoch, a) + }), + WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + return pool + }), + WithGCRemoverSleepInterval(1 * time.Second), + } + + sh = New(opts...) + + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init(context.Background())) + + t.Cleanup(func() { + require.NoError(t, sh.Close()) + }) + + cnr := cidtest.ID() + obj := testutil.GenerateObjectWithCID(cnr) + objID, _ := obj.ID() + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(objID) + + var putPrm PutPrm + putPrm.SetObject(obj) + + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + var getPrm GetPrm + getPrm.SetAddress(objectCore.AddressOf(obj)) + _, err = sh.Get(context.Background(), getPrm) + require.NoError(t, err, "failed to get") + + //inhume + var inhumePrm InhumePrm + inhumePrm.MarkAsGarbage(addr) + _, err = sh.Inhume(context.Background(), inhumePrm) + require.NoError(t, err, "failed to inhume") + _, err = sh.Get(context.Background(), getPrm) + require.Error(t, err, "get returned error") + require.True(t, IsErrNotFound(err), "invalid error type") + + //storageID + var metaStIDPrm meta.StorageIDPrm + metaStIDPrm.SetAddress(addr) + storageID, err := sh.metaBase.StorageID(context.Background(), metaStIDPrm) + require.NoError(t, err, "failed to get storage ID") + + //check existance in blobstore + var bsExisted common.ExistsPrm + bsExisted.Address = addr + bsExisted.StorageID = storageID.StorageID() + exRes, err := sh.blobStor.Exists(context.Background(), bsExisted) + require.NoError(t, err, "failed to check blobstore existance") + require.True(t, exRes.Exists, "invalid blobstore existance result") + + //drop from blobstor + var bsDeletePrm common.DeletePrm + bsDeletePrm.Address = addr + bsDeletePrm.StorageID = storageID.StorageID() + _, err = sh.blobStor.Delete(context.Background(), bsDeletePrm) + require.NoError(t, err, "failed to delete from blobstore") + + //check existance in blobstore + exRes, err = sh.blobStor.Exists(context.Background(), bsExisted) + require.NoError(t, err, "failed to check blobstore existance") + require.False(t, exRes.Exists, "invalid blobstore existance result") + + //get should return object not found + _, err = sh.Get(context.Background(), getPrm) + require.Error(t, err, "get returned no error") + require.True(t, IsErrNotFound(err), "invalid error type") +} diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index b0126fcd7..263a0ea4d 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -2,77 +2,30 @@ package shard_test import ( "context" - "path/filepath" + "errors" "testing" "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/panjf2000/ants/v2" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) -func Test_GCDropsLockedExpiredObject(t *testing.T) { - var sh *shard.Shard +func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { + t.Parallel() epoch := &epochState{ Value: 100, } - rootPath := t.TempDir() - opts := []shard.Option{ - shard.WithID(shard.NewIDFromBytes([]byte{})), - shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), - shard.WithBlobStorOptions( - blobstor.WithStorages([]blobstor.SubStorage{ - { - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), - blobovniczatree.WithBlobovniczaShallowDepth(2), - blobovniczatree.WithBlobovniczaShallowWidth(2)), - Policy: func(_ *objectSDK.Object, data []byte) bool { - return len(data) <= 1<<20 - }, - }, - { - Storage: fstree.New( - fstree.WithPath(filepath.Join(rootPath, "blob"))), - }, - }), - ), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(rootPath, "meta")), - meta.WithEpochState(epoch), - ), - shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { - sh.HandleDeletedLocks(addresses) - }), - shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { - sh.HandleExpiredLocks(ctx, epoch, a) - }), - shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { - pool, err := ants.NewPool(sz) - require.NoError(t, err) - - return pool - }), - } - - sh = shard.New(opts...) - require.NoError(t, sh.Open()) - require.NoError(t, sh.Init(context.Background())) + sh := newCustomShard(t, t.TempDir(), false, nil, nil, []meta.Option{meta.WithEpochState(epoch)}) t.Cleanup(func() { releaseShard(sh, t) @@ -120,3 +73,97 @@ func Test_GCDropsLockedExpiredObject(t *testing.T) { return shard.IsErrNotFound(err) }, 3*time.Second, 1*time.Second, "expired object must be deleted") } + +func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { + t.Parallel() + + epoch := &epochState{ + Value: 100, + } + + cnr := cidtest.ID() + parentID := oidtest.ID() + splitID := objectSDK.NewSplitID() + + var objExpirationAttr objectSDK.Attribute + objExpirationAttr.SetKey(objectV2.SysAttributeExpEpoch) + objExpirationAttr.SetValue("101") + + var lockExpirationAttr objectSDK.Attribute + lockExpirationAttr.SetKey(objectV2.SysAttributeExpEpoch) + lockExpirationAttr.SetValue("103") + + parent := testutil.GenerateObjectWithCID(cnr) + parent.SetID(parentID) + parent.SetPayload(nil) + parent.SetAttributes(objExpirationAttr) + + const childCount = 10 + children := make([]*objectSDK.Object, childCount) + childIDs := make([]oid.ID, childCount) + for i := range children { + children[i] = testutil.GenerateObjectWithCID(cnr) + if i != 0 { + children[i].SetPreviousID(childIDs[i-1]) + } + if i == len(children)-1 { + children[i].SetParent(parent) + } + children[i].SetSplitID(splitID) + children[i].SetPayload([]byte{byte(i), byte(i + 1), byte(i + 2)}) + childIDs[i], _ = children[i].ID() + } + + link := testutil.GenerateObjectWithCID(cnr) + link.SetParent(parent) + link.SetParentID(parentID) + link.SetSplitID(splitID) + link.SetChildren(childIDs...) + + linkID, _ := link.ID() + + sh := newCustomShard(t, t.TempDir(), false, nil, nil, []meta.Option{meta.WithEpochState(epoch)}) + + t.Cleanup(func() { + releaseShard(sh, t) + }) + + lock := testutil.GenerateObjectWithCID(cnr) + lock.SetType(objectSDK.TypeLock) + lock.SetAttributes(lockExpirationAttr) + lockID, _ := lock.ID() + + var putPrm shard.PutPrm + + for _, child := range children { + putPrm.SetObject(child) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + } + + putPrm.SetObject(link) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + err = sh.Lock(context.Background(), cnr, lockID, append(childIDs, parentID, linkID)) + require.NoError(t, err) + + putPrm.SetObject(lock) + _, err = sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + var getPrm shard.GetPrm + getPrm.SetAddress(objectCore.AddressOf(parent)) + + _, err = sh.Get(context.Background(), getPrm) + var splitInfoError *objectSDK.SplitInfoError + require.True(t, errors.As(err, &splitInfoError), "split info must be provided") + + epoch.Value = 105 + sh.NotificationChannel() <- shard.EventNewEpoch(epoch.Value) + + require.Eventually(t, func() bool { + _, err = sh.Get(context.Background(), getPrm) + return shard.IsErrNotFound(err) + }, 3*time.Second, 1*time.Second, "expired complex object must be deleted on epoch after lock expires") +} diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 5268ac790..3eb70784e 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -4,13 +4,13 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -66,6 +66,7 @@ func (r GetRes) HasMeta() bool { // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in shard. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Get", trace.WithAttributes( @@ -78,6 +79,10 @@ func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { s.m.RLock() defer s.m.RUnlock() + if s.info.Mode.Disabled() { + return GetRes{}, ErrShardDisabled + } + cb := func(stor *blobstor.BlobStor, id []byte) (*objectSDK.Object, error) { var getPrm common.GetPrm getPrm.Address = prm.addr diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index ea28c8e32..2db86c48a 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -17,11 +17,15 @@ import ( ) func TestShard_Get(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardGet(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardGet(t, true) }) } diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go index a15cdfdca..a0ec231af 100644 --- a/pkg/local_object_storage/shard/head.go +++ b/pkg/local_object_storage/shard/head.go @@ -3,8 +3,8 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index 11e7a8b04..7e336ea06 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -15,11 +15,15 @@ import ( ) func TestShard_Head(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardHead(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardHead(t, true) }) } diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index 992a86c01..e2ac423fd 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -55,6 +55,11 @@ func (s *Shard) UpdateID() (err error) { if s.hasWriteCache() { s.writeCache.SetLogger(s.log) } + s.metaBase.SetParentID(s.info.ID.String()) + s.blobStor.SetParentID(s.info.ID.String()) + if s.pilorama != nil { + s.pilorama.SetParentID(s.info.ID.String()) + } if len(id) != 0 { return nil diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 12a2900ac..6a2f9311d 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 0b4e51701..4151d6218 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -13,11 +13,15 @@ import ( ) func TestShard_Inhume(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardInhume(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardInhume(t, true) }) } diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index aaa1112cd..bc13c622b 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -7,8 +7,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -65,7 +68,13 @@ func (r ListWithCursorRes) Cursor() *Cursor { } // List returns all objects physically stored in the Shard. -func (s *Shard) List() (res SelectRes, err error) { +func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.List", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -73,12 +82,12 @@ func (s *Shard) List() (res SelectRes, err error) { return SelectRes{}, ErrDegradedMode } - lst, err := s.metaBase.Containers() + lst, err := s.metaBase.Containers(ctx) if err != nil { return res, fmt.Errorf("can't list stored containers: %w", err) } - filters := object.NewSearchFilters() + filters := objectSDK.NewSearchFilters() filters.AddPhyFilter() for i := range lst { @@ -86,7 +95,7 @@ func (s *Shard) List() (res SelectRes, err error) { sPrm.SetContainerID(lst[i]) sPrm.SetFilters(filters) - sRes, err := s.metaBase.Select(context.TODO(), sPrm) // consider making List in metabase + sRes, err := s.metaBase.Select(ctx, sPrm) // consider making List in metabase if err != nil { s.log.Debug(logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), @@ -101,12 +110,18 @@ func (s *Shard) List() (res SelectRes, err error) { return res, nil } -func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) { +func (s *Shard) ListContainers(ctx context.Context, _ ListContainersPrm) (ListContainersRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.ListContainers", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + if s.GetMode().NoMetabase() { return ListContainersRes{}, ErrDegradedMode } - containers, err := s.metaBase.Containers() + containers, err := s.metaBase.Containers(ctx) if err != nil { return ListContainersRes{}, fmt.Errorf("could not get list of containers: %w", err) } @@ -122,7 +137,14 @@ func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) { // // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. -func (s *Shard) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) { +func (s *Shard) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (ListWithCursorRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "shard.ListWithCursor", + trace.WithAttributes( + attribute.Int64("count", int64(prm.count)), + attribute.Bool("has_cursor", prm.cursor != nil), + )) + defer span.End() + if s.GetMode().NoMetabase() { return ListWithCursorRes{}, ErrDegradedMode } @@ -130,7 +152,7 @@ func (s *Shard) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) var metaPrm meta.ListPrm metaPrm.SetCount(prm.count) metaPrm.SetCursor(prm.cursor) - res, err := s.metaBase.ListWithCursor(metaPrm) + res, err := s.metaBase.ListWithCursor(ctx, metaPrm) if err != nil { return ListWithCursorRes{}, fmt.Errorf("could not get list of objects: %w", err) } diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index 8fac41a0f..63e7651c8 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -2,6 +2,7 @@ package shard_test import ( "context" + "sync" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -9,22 +10,23 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) func TestShard_List(t *testing.T) { - sh := newShard(t, false) - shWC := newShard(t, true) - - defer func() { - releaseShard(sh, t) - releaseShard(shWC, t) - }() + t.Parallel() t.Run("without write cache", func(t *testing.T) { + t.Parallel() + sh := newShard(t, false) + defer releaseShard(sh, t) testShardList(t, sh) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() + shWC := newShard(t, true) + defer releaseShard(shWC, t) testShardList(t, shWC) }) } @@ -33,32 +35,43 @@ func testShardList(t *testing.T, sh *shard.Shard) { const C = 10 const N = 5 + var mtx sync.Mutex objs := make(map[string]int) - var putPrm shard.PutPrm + var errG errgroup.Group + errG.SetLimit(C * N) for i := 0; i < C; i++ { - cnr := cidtest.ID() + errG.Go(func() error { + cnr := cidtest.ID() - for j := 0; j < N; j++ { - obj := testutil.GenerateObjectWithCID(cnr) - testutil.AddPayload(obj, 1<<2) + for j := 0; j < N; j++ { + errG.Go(func() error { + obj := testutil.GenerateObjectWithCID(cnr) + testutil.AddPayload(obj, 1<<2) - // add parent as virtual object, it must be ignored in List() - parent := testutil.GenerateObjectWithCID(cnr) - idParent, _ := parent.ID() - obj.SetParentID(idParent) - obj.SetParent(parent) + // add parent as virtual object, it must be ignored in List() + parent := testutil.GenerateObjectWithCID(cnr) + idParent, _ := parent.ID() + obj.SetParentID(idParent) + obj.SetParent(parent) - objs[object.AddressOf(obj).EncodeToString()] = 0 + mtx.Lock() + objs[object.AddressOf(obj).EncodeToString()] = 0 + mtx.Unlock() - putPrm.SetObject(obj) + var putPrm shard.PutPrm + putPrm.SetObject(obj) - _, err := sh.Put(context.Background(), putPrm) - require.NoError(t, err) - } + _, err := sh.Put(context.Background(), putPrm) + return err + }) + } + return nil + }) } + require.NoError(t, errG.Wait()) - res, err := sh.List() + res, err := sh.List(context.Background()) require.NoError(t, err) for _, objID := range res.AddressList() { diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go index cfbd94c5b..52186cbfd 100644 --- a/pkg/local_object_storage/shard/lock.go +++ b/pkg/local_object_storage/shard/lock.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index c577ae184..61f2bb9f0 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -15,7 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -23,6 +23,8 @@ import ( ) func TestShard_Lock(t *testing.T) { + t.Parallel() + var sh *shard.Shard rootPath := t.TempDir() @@ -36,7 +38,7 @@ func TestShard_Lock(t *testing.T) { blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(2), blobovniczatree.WithBlobovniczaShallowWidth(2)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return len(data) <= 1<<20 }, }, @@ -68,7 +70,7 @@ func TestShard_Lock(t *testing.T) { objID, _ := obj.ID() lock := testutil.GenerateObjectWithCID(cnr) - lock.SetType(object.TypeLock) + lock.SetType(objectSDK.TypeLock) lockID, _ := lock.ID() // put the object diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 1578c662b..ad0fc15ca 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -13,7 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -22,7 +22,8 @@ type metricsStore struct { objCounters map[string]uint64 cnrSize map[string]int64 pldSize int64 - readOnly bool + mode mode.Mode + errCounter int64 } func (m metricsStore) SetShardID(_ string) {} @@ -56,8 +57,8 @@ func (m metricsStore) DecObjectCounter(objectType string) { m.AddToObjectCounter(objectType, -1) } -func (m *metricsStore) SetReadonly(r bool) { - m.readOnly = r +func (m *metricsStore) SetMode(mode mode.Mode) { + m.mode = mode } func (m metricsStore) AddToContainerSize(cnr string, size int64) { @@ -68,20 +69,34 @@ func (m *metricsStore) AddToPayloadSize(size int64) { m.pldSize += size } +func (m *metricsStore) IncErrorCounter() { + m.errCounter += 1 +} + +func (m *metricsStore) ClearErrorCounter() { + m.errCounter = 0 +} + +func (m *metricsStore) DeleteShardMetrics() { + m.errCounter = 0 +} + const physical = "phy" const logical = "logic" func TestCounters(t *testing.T) { + t.Parallel() + dir := t.TempDir() sh, mm := shardWithMetrics(t, dir) sh.SetMode(mode.ReadOnly) - require.True(t, mm.readOnly) + require.Equal(t, mode.ReadOnly, mm.mode) sh.SetMode(mode.ReadWrite) - require.False(t, mm.readOnly) + require.Equal(t, mode.ReadWrite, mm.mode) const objNumber = 10 - oo := make([]*object.Object, objNumber) + oo := make([]*objectSDK.Object, objNumber) for i := 0; i < objNumber; i++ { oo[i] = testutil.GenerateObject() } @@ -225,7 +240,7 @@ func shardWithMetrics(t *testing.T, path string) (*shard.Shard, *metricsStore) { return sh, mm } -func addrFromObjs(oo []*object.Object) []oid.Address { +func addrFromObjs(oo []*objectSDK.Object) []oid.Address { aa := make([]oid.Address, len(oo)) for i := 0; i < len(oo); i++ { diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index 50c52accc..1bab57448 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -19,8 +19,8 @@ var ErrDegradedMode = logicerr.New("shard is in degraded mode") // Returns any error encountered that did not allow // setting shard mode. func (s *Shard) SetMode(m mode.Mode) error { - s.m.Lock() - defer s.m.Unlock() + unlock := s.lockExclusive() + defer unlock() return s.setMode(m) } @@ -56,15 +56,17 @@ func (s *Shard) setMode(m mode.Mode) error { } } - for i := range components { - if err := components[i].SetMode(m); err != nil { - return err + if !m.Disabled() { + for i := range components { + if err := components[i].SetMode(m); err != nil { + return err + } } } s.info.Mode = m if s.metricsWriter != nil { - s.metricsWriter.SetReadonly(s.info.Mode != mode.ReadWrite) + s.metricsWriter.SetMode(s.info.Mode) } s.log.Info(logs.ShardShardModeSetSuccessfully, diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go index 65b2b5c89..49c888d63 100644 --- a/pkg/local_object_storage/shard/mode/mode.go +++ b/pkg/local_object_storage/shard/mode/mode.go @@ -57,3 +57,7 @@ func (m Mode) NoMetabase() bool { func (m Mode) ReadOnly() bool { return m&ReadOnly != 0 } + +func (m Mode) Disabled() bool { + return m == Disabled +} diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go index 119910623..9832c9c84 100644 --- a/pkg/local_object_storage/shard/move.go +++ b/pkg/local_object_storage/shard/move.go @@ -3,9 +3,9 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index d7d4ae538..688b7aae7 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -4,12 +4,12 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -17,14 +17,14 @@ import ( // PutPrm groups the parameters of Put operation. type PutPrm struct { - obj *object.Object + obj *objectSDK.Object } // PutRes groups the resulting values of Put operation. type PutRes struct{} // SetObject is a Put option to set object to save. -func (p *PutPrm) SetObject(obj *object.Object) { +func (p *PutPrm) SetObject(obj *objectSDK.Object) { p.obj = obj } diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 06aea2f8a..e94482e36 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -4,13 +4,13 @@ import ( "context" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -29,7 +29,7 @@ type RngPrm struct { // RngRes groups the resulting values of GetRange operation. type RngRes struct { - obj *object.Object + obj *objectSDK.Object hasMeta bool } @@ -54,7 +54,7 @@ func (p *RngPrm) SetIgnoreMeta(ignore bool) { // Object returns the requested object part. // // Instance payload contains the requested range of the original object. -func (r RngRes) Object() *object.Object { +func (r RngRes) Object() *objectSDK.Object { return r.obj } @@ -71,7 +71,8 @@ func (r RngRes) HasMeta() bool { // Returns ErrRangeOutOfBounds if the requested object range is out of bounds. // Returns an error of type apistatus.ObjectNotFound if the requested object is missing. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. -// Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the objectSDK.ErrObjectIsExpired if the object is presented but already expired. +// Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.GetRange", trace.WithAttributes( @@ -86,7 +87,11 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { s.m.RLock() defer s.m.RUnlock() - cb := func(stor *blobstor.BlobStor, id []byte) (*object.Object, error) { + if s.info.Mode.Disabled() { + return RngRes{}, ErrShardDisabled + } + + cb := func(stor *blobstor.BlobStor, id []byte) (*objectSDK.Object, error) { var getRngPrm common.GetRangePrm getRngPrm.Address = prm.addr getRngPrm.Range.SetOffset(prm.off) @@ -98,13 +103,13 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { return nil, err } - obj := object.New() + obj := objectSDK.New() obj.SetPayload(res.Data) return obj, nil } - wc := func(c writecache.Cache) (*object.Object, error) { + wc := func(c writecache.Cache) (*objectSDK.Object, error) { res, err := c.Get(ctx, prm.addr) if err != nil { return nil, err @@ -117,7 +122,7 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { return nil, logicerr.Wrap(apistatus.ObjectOutOfRange{}) } - obj := object.New() + obj := objectSDK.New() obj.SetPayload(payload[from:to]) return obj, nil } diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index c95dbae98..9ef2106b0 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -22,11 +22,14 @@ import ( ) func TestShard_GetRange(t *testing.T) { + t.Parallel() t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardGetRange(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardGetRange(t, true) }) } @@ -84,7 +87,8 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { Storage: fstree.New( fstree.WithPath(filepath.Join(t.TempDir(), "blob"))), }, - })}) + })}, + nil) defer releaseShard(sh, t) for _, tc := range testCases { diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 9ad05f525..0b964ba2e 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -24,6 +24,8 @@ import ( ) func TestShardReload(t *testing.T) { + t.Parallel() + p := t.Name() defer os.RemoveAll(p) diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 7f776c18a..1615f5fbe 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -4,10 +4,10 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -16,7 +16,7 @@ import ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters } // SelectRes groups the resulting values of Select operation. @@ -30,7 +30,7 @@ func (p *SelectPrm) SetContainerID(cnr cid.ID) { } // SetFilters is a Select option to set the object filters. -func (p *SelectPrm) SetFilters(fs object.SearchFilters) { +func (p *SelectPrm) SetFilters(fs objectSDK.SearchFilters) { p.filters = fs } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 44ec54645..05799d236 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -3,6 +3,7 @@ package shard import ( "context" "sync" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -32,6 +33,9 @@ type Shard struct { metaBase *meta.DB tsSource TombstoneSource + + gcCancel atomic.Value + setModeRequested atomic.Bool } // Option represents Shard's constructor option. @@ -69,8 +73,14 @@ type MetricsWriter interface { // SetShardID must set (update) the shard identifier that will be used in // metrics. SetShardID(id string) - // SetReadonly must set shard readonly state. - SetReadonly(readonly bool) + // SetReadonly must set shard mode. + SetMode(mode mode.Mode) + // IncErrorCounter increment error counter. + IncErrorCounter() + // ClearErrorCounter clear error counter. + ClearErrorCounter() + // DeleteShardMetrics deletes shard metrics from registry. + DeleteShardMetrics() } type cfg struct { @@ -187,6 +197,13 @@ func WithWriteCacheOptions(opts ...writecache.Option) Option { } } +// WithExtraWriteCacheOptions returns option to add extra write cache options. +func WithExtraWriteCacheOptions(opts ...writecache.Option) Option { + return func(c *cfg) { + c.writeCacheOpts = append(c.writeCacheOpts, opts...) + } +} + // WithPiloramaOptions returns option to set internal write cache options. func WithPiloramaOptions(opts ...pilorama.Option) Option { return func(c *cfg) { @@ -210,12 +227,12 @@ func WithWriteCache(use bool) Option { } // hasWriteCache returns bool if write cache exists on shards. -func (s Shard) hasWriteCache() bool { +func (s *Shard) hasWriteCache() bool { return s.cfg.useWriteCache } -// needRefillMetabase returns true if metabase is needed to be refilled. -func (s Shard) needRefillMetabase() bool { +// NeedRefillMetabase returns true if metabase is needed to be refilled. +func (s *Shard) NeedRefillMetabase() bool { return s.cfg.refillMetabase } @@ -298,6 +315,13 @@ func WithMetricsWriter(v MetricsWriter) Option { } } +// WithGCMetrics returns option to specify storage of the GC metrics. +func WithGCMetrics(v GCMectrics) Option { + return func(c *cfg) { + c.gcCfg.metrics = v + } +} + // WithReportErrorFunc returns option to specify callback for handling storage-related errors // in the background workers. func WithReportErrorFunc(f func(selfID string, message string, err error)) Option { @@ -346,7 +370,7 @@ const ( logical = "logic" ) -func (s *Shard) updateMetrics() { +func (s *Shard) updateMetrics(ctx context.Context) { if s.cfg.metricsWriter != nil && !s.GetMode().NoMetabase() { cc, err := s.metaBase.ObjectCounters() if err != nil { @@ -360,7 +384,7 @@ func (s *Shard) updateMetrics() { s.cfg.metricsWriter.SetObjectCounter(physical, cc.Phy()) s.cfg.metricsWriter.SetObjectCounter(logical, cc.Logic()) - cnrList, err := s.metaBase.Containers() + cnrList, err := s.metaBase.Containers(ctx) if err != nil { s.log.Warn(logs.ShardMetaCantReadContainerList, zap.Error(err)) return @@ -410,3 +434,21 @@ func (s *Shard) addToPayloadSize(size int64) { s.cfg.metricsWriter.AddToPayloadSize(size) } } + +func (s *Shard) IncErrorCounter() { + if s.cfg.metricsWriter != nil { + s.cfg.metricsWriter.IncErrorCounter() + } +} + +func (s *Shard) ClearErrorCounter() { + if s.cfg.metricsWriter != nil { + s.cfg.metricsWriter.ClearErrorCounter() + } +} + +func (s *Shard) DeleteShardMetrics() { + if s.cfg.metricsWriter != nil { + s.cfg.metricsWriter.DeleteShardMetrics() + } +} diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index fea342766..a9a8e4ea7 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -4,6 +4,7 @@ import ( "context" "path/filepath" "testing" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" @@ -12,8 +13,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -29,11 +33,13 @@ func (s epochState) CurrentEpoch() uint64 { func newShard(t testing.TB, enableWriteCache bool) *shard.Shard { return newCustomShard(t, t.TempDir(), enableWriteCache, + nil, nil, nil) } -func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts []writecache.Option, bsOpts []blobstor.Option) *shard.Shard { +func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts []writecache.Option, bsOpts []blobstor.Option, metaOptions []meta.Option) *shard.Shard { + var sh *shard.Shard if enableWriteCache { rootPath = filepath.Join(rootPath, "wc") } else { @@ -50,7 +56,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return len(data) <= 1<<20 }, }, @@ -67,8 +73,9 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions(bsOpts...), shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(rootPath, "meta")), - meta.WithEpochState(epochState{}), + append([]meta.Option{ + meta.WithPath(filepath.Join(rootPath, "meta")), meta.WithEpochState(epochState{})}, + metaOptions...)..., ), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), shard.WithWriteCache(enableWriteCache), @@ -77,9 +84,21 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts []writecache.Option{writecache.WithPath(filepath.Join(rootPath, "wcache"))}, wcOpts...)..., ), + shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(addresses) + }), + shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { + sh.HandleExpiredLocks(ctx, epoch, a) + }), + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + return pool + }), + shard.WithGCRemoverSleepInterval(1 * time.Millisecond), } - sh := shard.New(opts...) + sh = shard.New(opts...) require.NoError(t, sh.Open()) require.NoError(t, sh.Init(context.Background())) diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 714811b7e..5fe9fd7e9 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -12,9 +12,12 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) func TestWriteCacheObjectLoss(t *testing.T) { + t.Parallel() + const ( smallSize = 1024 objCount = 100 @@ -37,18 +40,22 @@ func TestWriteCacheObjectLoss(t *testing.T) { writecache.WithSmallObjectSize(smallSize), writecache.WithMaxObjectSize(smallSize * 2)} - sh := newCustomShard(t, dir, true, wcOpts, nil) - - var putPrm shard.PutPrm + sh := newCustomShard(t, dir, true, wcOpts, nil, nil) + var errG errgroup.Group for i := range objects { - putPrm.SetObject(objects[i]) - _, err := sh.Put(context.Background(), putPrm) - require.NoError(t, err) + obj := objects[i] + errG.Go(func() error { + var putPrm shard.PutPrm + putPrm.SetObject(obj) + _, err := sh.Put(context.Background(), putPrm) + return err + }) } + require.NoError(t, errG.Wait()) require.NoError(t, sh.Close()) - sh = newCustomShard(t, dir, true, wcOpts, nil) + sh = newCustomShard(t, dir, true, wcOpts, nil, nil) defer releaseShard(sh, t) var getPrm shard.GetPrm diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index d5b3b67bf..7e2c80152 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -4,9 +4,9 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -159,7 +159,7 @@ func (s *Shard) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, n } // TreeGetChildren implements the pilorama.Forest interface. -func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { +func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetChildren", trace.WithAttributes( attribute.String("shard_id", s.ID().String()), @@ -255,6 +255,29 @@ func (s *Shard) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { return s.pilorama.TreeList(ctx, cid) } +func (s *Shard) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeHeight", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return 0, ErrDegradedMode + } + + if s.pilorama == nil { + return 0, ErrPiloramaDisabled + } + return s.pilorama.TreeHeight(ctx, cid, treeID) +} + // TreeExists implements the pilorama.Forest interface. func (s *Shard) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeExists", diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 245eb4c70..7ce279c54 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/local_object_storage/util/splitinfo.go b/pkg/local_object_storage/util/splitinfo.go index 4a6d22268..6ae1c3e46 100644 --- a/pkg/local_object_storage/util/splitinfo.go +++ b/pkg/local_object_storage/util/splitinfo.go @@ -1,12 +1,12 @@ package util import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) // MergeSplitInfo ignores conflicts and rewrites `to` with non empty values // from `from`. -func MergeSplitInfo(from, to *object.SplitInfo) *object.SplitInfo { +func MergeSplitInfo(from, to *objectSDK.SplitInfo) *objectSDK.SplitInfo { to.SetSplitID(from.SplitID()) // overwrite SplitID and ignore conflicts if lp, ok := from.LastPart(); ok { diff --git a/pkg/local_object_storage/util/splitinfo_test.go b/pkg/local_object_storage/util/splitinfo_test.go index a0626db28..642fef3b8 100644 --- a/pkg/local_object_storage/util/splitinfo_test.go +++ b/pkg/local_object_storage/util/splitinfo_test.go @@ -5,7 +5,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -15,7 +15,7 @@ func TestMergeSplitInfo(t *testing.T) { uid, err := uuid.NewUUID() require.NoError(t, err) - splitID := object.NewSplitID() + splitID := objectSDK.NewSplitID() splitID.SetUUID(uid) var rawLinkID, rawLastID [32]byte @@ -30,35 +30,35 @@ func TestMergeSplitInfo(t *testing.T) { require.NoError(t, err) lastID.SetSHA256(rawLastID) - target := object.NewSplitInfo() // target is SplitInfo struct with all fields set + target := objectSDK.NewSplitInfo() // target is SplitInfo struct with all fields set target.SetSplitID(splitID) target.SetLastPart(lastID) target.SetLink(linkID) t.Run("merge empty", func(t *testing.T) { - to := object.NewSplitInfo() + to := objectSDK.NewSplitInfo() result := util.MergeSplitInfo(target, to) require.Equal(t, result, target) }) t.Run("merge link", func(t *testing.T) { - from := object.NewSplitInfo() + from := objectSDK.NewSplitInfo() from.SetSplitID(splitID) from.SetLastPart(lastID) - to := object.NewSplitInfo() + to := objectSDK.NewSplitInfo() to.SetLink(linkID) result := util.MergeSplitInfo(from, to) require.Equal(t, result, target) }) t.Run("merge last", func(t *testing.T) { - from := object.NewSplitInfo() + from := objectSDK.NewSplitInfo() from.SetSplitID(splitID) from.SetLink(linkID) - to := object.NewSplitInfo() + to := objectSDK.NewSplitInfo() to.SetLastPart(lastID) result := util.MergeSplitInfo(from, to) diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index c1aab9e5a..aeab88b0b 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -2,10 +2,12 @@ package writecache import ( "context" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -22,6 +24,13 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { )) defer span.End() + deleted := false + storageType := StorageTypeUndefined + startedAt := time.Now() + defer func() { + c.metrics.Delete(time.Since(startedAt), deleted, storageType) + }() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { @@ -30,15 +39,15 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { saddr := addr.EncodeToString() - // Check disk cache. - var has int + var dataSize int _ = c.db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) - has = len(b.Get([]byte(saddr))) + dataSize = len(b.Get([]byte(saddr))) return nil }) - if 0 < has { + if dataSize > 0 { + storageType = StorageTypeDB err := c.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) err := b.Delete([]byte(saddr)) @@ -52,10 +61,12 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("db DELETE"), ) + deleted = true c.objCounters.DecDB() return nil } + storageType = StorageTypeFSTree _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { storagelog.Write(c.log, @@ -64,7 +75,8 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.OpField("fstree DELETE"), ) c.objCounters.DecFS() + deleted = true } - return err + return metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 04fcccede..243be4627 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -6,13 +6,14 @@ import ( "errors" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/mr-tron/base58" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -35,14 +36,24 @@ const ( // runFlushLoop starts background workers which periodically flush objects to the blobstor. func (c *cache) runFlushLoop() { + ctx, cancel := context.WithCancel(context.Background()) + + ch := c.closeCh + c.wg.Add(1) + go func() { + <-ch + cancel() + c.wg.Done() + }() + for i := 0; i < c.workersCount; i++ { c.wg.Add(1) - go c.flushWorker(i) + go c.workerFlushSmall() } c.wg.Add(1) go func() { - c.flushBigObjects(context.TODO()) + c.workerFlushBig(ctx) c.wg.Done() }() @@ -56,7 +67,7 @@ func (c *cache) runFlushLoop() { for { select { case <-tt.C: - c.flushDB() + c.flushSmallObjects() tt.Reset(defaultFlushInterval) case <-c.closeCh: return @@ -65,7 +76,7 @@ func (c *cache) runFlushLoop() { }() } -func (c *cache) flushDB() { +func (c *cache) flushSmallObjects() { var lastKey []byte var m []objectInfo for { @@ -78,7 +89,7 @@ func (c *cache) flushDB() { m = m[:0] c.modeMtx.RLock() - if c.readOnly() || !c.initialized.Load() { + if c.readOnly() { c.modeMtx.RUnlock() time.Sleep(time.Second) continue @@ -117,11 +128,7 @@ func (c *cache) flushDB() { var count int for i := range m { - if c.flushed.Contains(m[i].addr) { - continue - } - - obj := object.New() + obj := objectSDK.New() if err := obj.Unmarshal(m[i].data); err != nil { continue } @@ -148,7 +155,7 @@ func (c *cache) flushDB() { } } -func (c *cache) flushBigObjects(ctx context.Context) { +func (c *cache) workerFlushBig(ctx context.Context) { tick := time.NewTicker(defaultFlushInterval * 10) for { select { @@ -157,9 +164,6 @@ func (c *cache) flushBigObjects(ctx context.Context) { if c.readOnly() { c.modeMtx.RUnlock() break - } else if !c.initialized.Load() { - c.modeMtx.RUnlock() - continue } _ = c.flushFSTree(ctx, true) @@ -187,30 +191,26 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { prm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { sAddr := addr.EncodeToString() - if _, ok := c.store.flushed.Peek(sAddr); ok { - return nil - } - data, err := f() if err != nil { - c.reportFlushError("can't read a file", sAddr, err) + c.reportFlushError("can't read a file", sAddr, metaerr.Wrap(err)) if ignoreErrors { return nil } return err } - var obj object.Object + var obj objectSDK.Object err = obj.Unmarshal(data) if err != nil { - c.reportFlushError("can't unmarshal an object", sAddr, err) + c.reportFlushError("can't unmarshal an object", sAddr, metaerr.Wrap(err)) if ignoreErrors { return nil } return err } - err = c.flushObject(ctx, &obj, data) + err = c.flushObject(ctx, &obj, data, StorageTypeFSTree) if err != nil { if ignoreErrors { return nil @@ -218,21 +218,19 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - // mark object as flushed - c.flushed.Add(sAddr, false) - + c.deleteFromDisk(ctx, []string{sAddr}) return nil } - _, err := c.fsTree.Iterate(prm) + _, err := c.fsTree.Iterate(ctx, prm) return err } -// flushWorker writes objects to the main storage. -func (c *cache) flushWorker(_ int) { +// workerFlushSmall writes small objects to the main storage. +func (c *cache) workerFlushSmall() { defer c.wg.Done() - var obj *object.Object + var obj *objectSDK.Object for { // Give priority to direct put. select { @@ -241,15 +239,24 @@ func (c *cache) flushWorker(_ int) { return } - err := c.flushObject(context.TODO(), obj, nil) - if err == nil { - c.flushed.Add(objectCore.AddressOf(obj).EncodeToString(), true) + err := c.flushObject(context.TODO(), obj, nil, StorageTypeDB) + if err != nil { + // Error is handled in flushObject. + continue } + + c.deleteFromDB([]string{objectCore.AddressOf(obj).EncodeToString()}) } } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte) error { +func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st StorageType) error { + var err error + + defer func() { + c.metrics.Flush(err == nil, st) + }() + addr := objectCore.AddressOf(obj) var prm common.PutPrm @@ -306,28 +313,24 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { cs := b.Cursor() for k, data := cs.Seek(nil); k != nil; k, data = cs.Next() { sa := string(k) - if _, ok := c.flushed.Peek(sa); ok { - continue - } - if err := addr.DecodeString(sa); err != nil { - c.reportFlushError("can't decode object address from the DB", sa, err) + c.reportFlushError("can't decode object address from the DB", sa, metaerr.Wrap(err)) if ignoreErrors { continue } return err } - var obj object.Object + var obj objectSDK.Object if err := obj.Unmarshal(data); err != nil { - c.reportFlushError("can't unmarshal an object from the DB", sa, err) + c.reportFlushError("can't unmarshal an object from the DB", sa, metaerr.Wrap(err)) if ignoreErrors { continue } return err } - if err := c.flushObject(ctx, &obj, data); err != nil { + if err := c.flushObject(ctx, &obj, data, StorageTypeDB); err != nil { return err } } diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 2cec07081..2223bef02 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -4,8 +4,8 @@ import ( "context" "os" "path/filepath" + "sync/atomic" "testing" - "time" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -15,22 +15,20 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" versionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap/zaptest" ) type objectPair struct { addr oid.Address - obj *object.Object + obj *objectSDK.Object } func TestFlush(t *testing.T) { @@ -109,22 +107,9 @@ func TestFlush(t *testing.T) { require.NoError(t, bs.SetMode(mode.ReadWrite)) require.NoError(t, mb.SetMode(mode.ReadWrite)) - wc.(*cache).flushed.Add(objects[0].addr.EncodeToString(), true) - wc.(*cache).flushed.Add(objects[1].addr.EncodeToString(), false) - require.NoError(t, wc.Flush(context.Background(), false)) - for i := 0; i < 2; i++ { - var mPrm meta.GetPrm - mPrm.SetAddress(objects[i].addr) - _, err := mb.Get(context.Background(), mPrm) - require.Error(t, err) - - _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) - require.Error(t, err) - } - - check(t, mb, bs, objects[2:]) + check(t, mb, bs, objects) }) t.Run("flush on moving to degraded mode", func(t *testing.T) { @@ -138,30 +123,16 @@ func TestFlush(t *testing.T) { require.NoError(t, wc.SetMode(mode.ReadOnly)) require.NoError(t, bs.SetMode(mode.ReadWrite)) require.NoError(t, mb.SetMode(mode.ReadWrite)) - - wc.(*cache).flushed.Add(objects[0].addr.EncodeToString(), true) - wc.(*cache).flushed.Add(objects[1].addr.EncodeToString(), false) - require.NoError(t, wc.SetMode(mode.Degraded)) - for i := 0; i < 2; i++ { - var mPrm meta.GetPrm - mPrm.SetAddress(objects[i].addr) - _, err := mb.Get(context.Background(), mPrm) - require.Error(t, err) - - _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) - require.Error(t, err) - } - - check(t, mb, bs, objects[2:]) + check(t, mb, bs, objects) }) t.Run("ignore errors", func(t *testing.T) { testIgnoreErrors := func(t *testing.T, f func(*cache)) { var errCount atomic.Uint32 wc, bs, mb := newCache(t, WithReportErrorFunc(func(message string, err error) { - errCount.Inc() + errCount.Add(1) })) objects := putObjects(t, wc) f(wc.(*cache)) @@ -223,67 +194,6 @@ func TestFlush(t *testing.T) { }) }) }) - - t.Run("on init", func(t *testing.T) { - wc, bs, mb := newCache(t) - objects := []objectPair{ - // removed - putObject(t, wc, 1), - putObject(t, wc, smallSize+1), - // not found - putObject(t, wc, 1), - putObject(t, wc, smallSize+1), - // ok - putObject(t, wc, 1), - putObject(t, wc, smallSize+1), - } - - require.NoError(t, wc.Close()) - require.NoError(t, bs.SetMode(mode.ReadWrite)) - require.NoError(t, mb.SetMode(mode.ReadWrite)) - - for i := range objects { - var prm meta.PutPrm - prm.SetObject(objects[i].obj) - _, err := mb.Put(context.Background(), prm) - require.NoError(t, err) - } - - var inhumePrm meta.InhumePrm - inhumePrm.SetAddresses(objects[0].addr, objects[1].addr) - inhumePrm.SetTombstoneAddress(oidtest.Address()) - _, err := mb.Inhume(context.Background(), inhumePrm) - require.NoError(t, err) - - var deletePrm meta.DeletePrm - deletePrm.SetAddresses(objects[2].addr, objects[3].addr) - _, err = mb.Delete(context.Background(), deletePrm) - require.NoError(t, err) - - require.NoError(t, bs.SetMode(mode.ReadOnly)) - require.NoError(t, mb.SetMode(mode.ReadOnly)) - - // Open in read-only: no error, nothing is removed. - require.NoError(t, wc.Open(true)) - initWC(t, wc) - for i := range objects { - _, err := wc.Get(context.Background(), objects[i].addr) - require.NoError(t, err, i) - } - require.NoError(t, wc.Close()) - - // Open in read-write: no error, something is removed. - require.NoError(t, wc.Open(false)) - initWC(t, wc) - for i := range objects { - _, err := wc.Get(context.Background(), objects[i].addr) - if i < 2 { - require.ErrorAs(t, err, new(apistatus.ObjectNotFound), i) - } else { - require.NoError(t, err, i) - } - } - }) } func putObject(t *testing.T, c Cache, size int) objectPair { @@ -301,14 +211,14 @@ func putObject(t *testing.T, c Cache, size int) objectPair { } -func newObject(t *testing.T, size int) (*object.Object, []byte) { - obj := object.New() +func newObject(t *testing.T, size int) (*objectSDK.Object, []byte) { + obj := objectSDK.New() ver := versionSDK.Current() obj.SetID(oidtest.ID()) obj.SetOwnerID(usertest.ID()) obj.SetContainerID(cidtest.ID()) - obj.SetType(object.TypeRegular) + obj.SetType(objectSDK.TypeRegular) obj.SetVersion(&ver) obj.SetPayloadChecksum(checksumtest.Checksum()) obj.SetPayloadHomomorphicHash(checksumtest.Checksum()) @@ -321,11 +231,6 @@ func newObject(t *testing.T, size int) (*object.Object, []byte) { func initWC(t *testing.T, wc Cache) { require.NoError(t, wc.Init()) - - require.Eventually(t, func() bool { - rawWc := wc.(*cache) - return rawWc.initialized.Load() - }, 100*time.Second, 1*time.Millisecond) } type dummyEpoch struct{} diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index 6af1bd181..2546bada9 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -2,10 +2,12 @@ package writecache import ( "context" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -27,10 +29,23 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e )) defer span.End() + obj, err := c.getInternal(ctx, saddr, addr) + return obj, metaerr.Wrap(err) +} + +func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) { + found := false + storageType := StorageTypeUndefined + startedAt := time.Now() + defer func() { + c.metrics.Get(time.Since(startedAt), found, storageType) + }() + value, err := Get(c.db, []byte(saddr)) if err == nil { obj := objectSDK.New() - c.flushed.Get(saddr) + found = true + storageType = StorageTypeDB return obj, obj.Unmarshal(value) } @@ -39,7 +54,8 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) } - c.flushed.Get(saddr) + found = true + storageType = StorageTypeFSTree return res.Object, nil } @@ -47,15 +63,17 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e // // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache. func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { + saddr := addr.EncodeToString() + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Head", trace.WithAttributes( - attribute.String("address", addr.EncodeToString()), + attribute.String("address", saddr), )) defer span.End() - obj, err := c.Get(ctx, addr) + obj, err := c.getInternal(ctx, saddr, addr) if err != nil { - return nil, err + return nil, metaerr.Wrap(err) } return obj.CutPayload(), nil @@ -79,5 +97,5 @@ func Get(db *bbolt.DB, key []byte) ([]byte, error) { value = slice.Copy(value) return nil }) - return value, err + return value, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/writecache/init.go b/pkg/local_object_storage/writecache/init.go deleted file mode 100644 index 2ca8cceef..000000000 --- a/pkg/local_object_storage/writecache/init.go +++ /dev/null @@ -1,192 +0,0 @@ -package writecache - -import ( - "context" - "errors" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.etcd.io/bbolt" - "go.uber.org/zap" -) - -func (c *cache) initFlushMarks(ctx context.Context) { - var localWG sync.WaitGroup - - localWG.Add(1) - go func() { - defer localWG.Done() - - c.fsTreeFlushMarkUpdate(ctx) - }() - - localWG.Add(1) - go func() { - defer localWG.Done() - - c.dbFlushMarkUpdate(ctx) - }() - - c.initWG.Add(1) - c.wg.Add(1) - go func() { - defer c.wg.Done() - defer c.initWG.Done() - - localWG.Wait() - - select { - case <-c.stopInitCh: - return - case <-c.closeCh: - return - default: - } - - c.initialized.Store(true) - }() -} - -var errStopIter = errors.New("stop iteration") - -func (c *cache) fsTreeFlushMarkUpdate(ctx context.Context) { - c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInFSTree) - - var prm common.IteratePrm - prm.LazyHandler = func(addr oid.Address, _ func() ([]byte, error)) error { - select { - case <-c.closeCh: - return errStopIter - case <-c.stopInitCh: - return errStopIter - default: - } - - flushed, needRemove := c.flushStatus(ctx, addr) - if flushed { - c.store.flushed.Add(addr.EncodeToString(), true) - if needRemove { - var prm common.DeletePrm - prm.Address = addr - - _, err := c.fsTree.Delete(ctx, prm) - if err == nil { - storagelog.Write(c.log, - storagelog.AddressField(addr), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("fstree DELETE"), - ) - } - } - } - return nil - } - - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - - _, _ = c.fsTree.Iterate(prm) - - c.log.Info(logs.WritecacheFinishedUpdatingFSTreeFlushMarks) -} - -func (c *cache) dbFlushMarkUpdate(ctx context.Context) { - c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInDatabase) - - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - - var m []string - var indices []int - var lastKey []byte - var batchSize = flushBatchSize - for { - select { - case <-c.closeCh: - return - case <-c.stopInitCh: - return - default: - } - - m = m[:0] - indices = indices[:0] - - // We put objects in batches of fixed size to not interfere with main put cycle a lot. - _ = c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - cs := b.Cursor() - for k, _ := cs.Seek(lastKey); k != nil && len(m) < batchSize; k, _ = cs.Next() { - m = append(m, string(k)) - } - return nil - }) - - var addr oid.Address - for i := range m { - if err := addr.DecodeString(m[i]); err != nil { - continue - } - - flushed, needRemove := c.flushStatus(ctx, addr) - if flushed { - c.store.flushed.Add(addr.EncodeToString(), true) - if needRemove { - indices = append(indices, i) - } - } - } - - if len(m) == 0 { - break - } - - err := c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - for _, j := range indices { - if err := b.Delete([]byte(m[j])); err != nil { - return err - } - } - return nil - }) - if err == nil { - for _, j := range indices { - storagelog.Write(c.log, - zap.String("address", m[j]), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("db DELETE"), - ) - } - } - lastKey = append([]byte(m[len(m)-1]), 0) - } - - c.log.Info(logs.WritecacheFinishedUpdatingFlushMarks) -} - -// flushStatus returns info about the object state in the main storage. -// First return value is true iff object exists. -// Second return value is true iff object can be safely removed. -func (c *cache) flushStatus(ctx context.Context, addr oid.Address) (bool, bool) { - var existsPrm meta.ExistsPrm - existsPrm.SetAddress(addr) - - _, err := c.metabase.Exists(ctx, existsPrm) - if err != nil { - needRemove := errors.Is(err, meta.ErrObjectIsExpired) || errors.As(err, new(apistatus.ObjectAlreadyRemoved)) - return needRemove, needRemove - } - - var prm meta.StorageIDPrm - prm.SetAddress(addr) - - mRes, _ := c.metabase.StorageID(ctx, prm) - res, err := c.blobstor.Exists(ctx, common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()}) - return err == nil && res.Exists, false -} diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/iterate.go index 228dd2597..5349c069c 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -12,65 +12,6 @@ import ( // ErrNoDefaultBucket is returned by IterateDB when default bucket for objects is missing. var ErrNoDefaultBucket = errors.New("no default bucket") -// IterationPrm contains iteration parameters. -type IterationPrm struct { - handler func([]byte) error - ignoreErrors bool -} - -// WithHandler sets a callback to be executed on every object. -func (p *IterationPrm) WithHandler(f func([]byte) error) { - p.handler = f -} - -// WithIgnoreErrors sets a flag indicating that errors should be ignored. -func (p *IterationPrm) WithIgnoreErrors(ignore bool) { - p.ignoreErrors = ignore -} - -// Iterate iterates over all objects present in write cache. -// This is very difficult to do correctly unless write-cache is put in read-only mode. -// Thus we silently fail if shard is not in read-only mode to avoid reporting misleading results. -func (c *cache) Iterate(prm IterationPrm) error { - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - if !c.readOnly() { - return nil - } - - err := c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - return b.ForEach(func(k, data []byte) error { - if _, ok := c.flushed.Peek(string(k)); ok { - return nil - } - return prm.handler(data) - }) - }) - if err != nil { - return err - } - - var fsPrm common.IteratePrm - fsPrm.IgnoreErrors = prm.ignoreErrors - fsPrm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { - if _, ok := c.flushed.Peek(addr.EncodeToString()); ok { - return nil - } - data, err := f() - if err != nil { - if prm.ignoreErrors { - return nil - } - return err - } - return prm.handler(data) - } - - _, err = c.fsTree.Iterate(fsPrm) - return err -} - // IterateDB iterates over all objects stored in bbolt.DB instance and passes them to f until error return. // It is assumed that db is an underlying database of some WriteCache instance. // @@ -78,7 +19,7 @@ func (c *cache) Iterate(prm IterationPrm) error { // // DB must not be nil and should be opened. func IterateDB(db *bbolt.DB, f func(oid.Address) error) error { - return db.View(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) if b == nil { return ErrNoDefaultBucket @@ -94,5 +35,5 @@ func IterateDB(db *bbolt.DB, f func(oid.Address) error) error { return f(addr) }) - }) + })) } diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go new file mode 100644 index 000000000..957bf2770 --- /dev/null +++ b/pkg/local_object_storage/writecache/metrics.go @@ -0,0 +1,52 @@ +package writecache + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) + +type StorageType string + +func (t StorageType) String() string { + return string(t) +} + +const ( + StorageTypeUndefined StorageType = "null" + StorageTypeDB StorageType = "db" + StorageTypeFSTree StorageType = "fstree" +) + +type Metrics interface { + Get(d time.Duration, success bool, st StorageType) + Delete(d time.Duration, success bool, st StorageType) + Put(d time.Duration, success bool, st StorageType) + Flush(success bool, st StorageType) + Evict(st StorageType) + + SetEstimateSize(db, fstree uint64) + SetMode(m mode.Mode) + SetActualCounters(db, fstree uint64) + Close() +} + +type metricsStub struct{} + +func (s *metricsStub) Get(time.Duration, bool, StorageType) {} + +func (s *metricsStub) Delete(time.Duration, bool, StorageType) {} + +func (s *metricsStub) Put(time.Duration, bool, StorageType) {} + +func (s *metricsStub) SetEstimateSize(uint64, uint64) {} + +func (s *metricsStub) SetMode(mode.Mode) {} + +func (s *metricsStub) SetActualCounters(uint64, uint64) {} + +func (s *metricsStub) Flush(bool, StorageType) {} + +func (s *metricsStub) Evict(StorageType) {} + +func (s *metricsStub) Close() {} diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 14f8af49e..bdbbec7c9 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -5,10 +5,10 @@ import ( "fmt" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -29,7 +29,11 @@ func (c *cache) SetMode(m mode.Mode) error { )) defer span.End() - return c.setMode(ctx, m) + err := c.setMode(ctx, m) + if err == nil { + c.metrics.SetMode(m) + } + return err } // setMode applies new mode. Must be called with cache.modeMtx lock taken. @@ -37,22 +41,6 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode) error { var err error turnOffMeta := m.NoMetabase() - if !c.initialized.Load() { - close(c.stopInitCh) - - c.initWG.Wait() - c.stopInitCh = make(chan struct{}) - - defer func() { - if err == nil && !turnOffMeta { - c.initFlushMarks(ctx) - } - }() - } - - c.modeMtx.Lock() - defer c.modeMtx.Unlock() - if turnOffMeta && !c.mode.NoMetabase() { err = c.flush(ctx, true) if err != nil { diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index 3434e9355..bea40aa36 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -60,6 +60,8 @@ type options struct { reportError func(string, error) // openFile is the function called internally by bbolt to open database files. Useful for hermetic testing. openFile func(string, int, fs.FileMode) (*os.File, error) + // metrics is metrics implementation + metrics Metrics } // WithLogger sets logger. @@ -164,3 +166,10 @@ func WithOpenFile(f func(string, int, fs.FileMode) (*os.File, error)) Option { o.openFile = f } } + +// WithMetrics sets metrics implementation. +func WithMetrics(metrics Metrics) Option { + return func(o *options) { + o.metrics = metrics + } +} diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index e2535d9e2..619b2bd26 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -3,10 +3,12 @@ package writecache import ( "context" "errors" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -33,12 +35,17 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro )) defer span.End() + startedAt := time.Now() + added := false + storageType := StorageTypeUndefined + defer func() { + c.metrics.Put(time.Since(startedAt), added, storageType) + }() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { return common.PutRes{}, ErrReadOnly - } else if !c.initialized.Load() { - return common.PutRes{}, ErrNotInitialized } sz := uint64(len(prm.RawData)) @@ -53,9 +60,20 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro } if sz <= c.smallObjectSize { - return common.PutRes{}, c.putSmall(oi) + storageType = StorageTypeDB + err := c.putSmall(oi) + if err == nil { + added = true + } + return common.PutRes{}, err } - return common.PutRes{}, c.putBig(ctx, oi.addr, prm) + + storageType = StorageTypeFSTree + err := c.putBig(ctx, oi.addr, prm) + if err == nil { + added = true + } + return common.PutRes{}, metaerr.Wrap(err) } // putSmall persists small objects to the write-cache database and @@ -78,7 +96,7 @@ func (c *cache) putSmall(obj objectInfo) error { ) c.objCounters.IncDB() } - return nil + return err } // putBig writes object to FSTree and pushes it to the flush workers queue. diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index 1ba5a4bd3..14103e626 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -2,13 +2,17 @@ package writecache import ( "fmt" + "math" + "sync/atomic" "go.etcd.io/bbolt" - "go.uber.org/atomic" ) func (c *cache) estimateCacheSize() uint64 { - return c.objCounters.DB()*c.smallObjectSize + c.objCounters.FS()*c.maxObjectSize + db := c.objCounters.DB() * c.smallObjectSize + fstree := c.objCounters.FS() * c.maxObjectSize + c.metrics.SetEstimateSize(db, fstree) + return db + fstree } func (c *cache) incSizeDB(sz uint64) uint64 { @@ -24,11 +28,11 @@ type counters struct { } func (x *counters) IncDB() { - x.cDB.Inc() + x.cDB.Add(1) } func (x *counters) DecDB() { - x.cDB.Dec() + x.cDB.Add(math.MaxUint64) } func (x *counters) DB() uint64 { @@ -36,11 +40,11 @@ func (x *counters) DB() uint64 { } func (x *counters) IncFS() { - x.cFS.Inc() + x.cFS.Add(1) } func (x *counters) DecFS() { - x.cFS.Dec() + x.cFS.Add(math.MaxUint64) } func (x *counters) FS() uint64 { @@ -67,6 +71,7 @@ func (c *cache) initCounters() error { c.objCounters.cDB.Store(inDB) c.objCounters.cFS.Store(inFS) + c.metrics.SetActualCounters(inDB, inFS) return nil } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index aeae752e3..3bd3813d1 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -13,8 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - lru "github.com/hashicorp/golang-lru/v2" - "github.com/hashicorp/golang-lru/v2/simplelru" "go.etcd.io/bbolt" "go.uber.org/zap" ) @@ -22,19 +20,7 @@ import ( // store represents persistent storage with in-memory LRU cache // for flushed items on top of it. type store struct { - maxFlushedMarksCount int - maxRemoveBatchSize int - - // flushed contains addresses of objects that were already flushed to the main storage. - // We use LRU cache instead of map here to facilitate removing of unused object in favour of - // frequently read ones. - // MUST NOT be used inside bolt db transaction because it's eviction handler - // removes untracked items from the database. - flushed simplelru.LRUCache[string, bool] - db *bbolt.DB - - dbKeysToRemove []string - fsKeysToRemove []string + db *bbolt.DB } const dbName = "small.bolt" @@ -73,35 +59,9 @@ func (c *cache) openStore(readOnly bool) error { return fmt.Errorf("could not open FSTree: %w", err) } - // Write-cache can be opened multiple times during `SetMode`. - // flushed map must not be re-created in this case. - if c.flushed == nil { - c.flushed, _ = lru.NewWithEvict[string, bool](c.maxFlushedMarksCount, c.removeFlushed) - } - - c.initialized.Store(false) - return nil } -// removeFlushed removes an object from the writecache. -// To minimize interference with the client operations, the actual removal -// is done in batches. -// It is not thread-safe and is used only as an evict callback to LRU cache. -func (c *cache) removeFlushed(key string, value bool) { - fromDatabase := value - if fromDatabase { - c.dbKeysToRemove = append(c.dbKeysToRemove, key) - } else { - c.fsKeysToRemove = append(c.fsKeysToRemove, key) - } - - if len(c.dbKeysToRemove)+len(c.fsKeysToRemove) >= c.maxRemoveBatchSize { - c.dbKeysToRemove = c.deleteFromDB(c.dbKeysToRemove) - c.fsKeysToRemove = c.deleteFromDisk(c.fsKeysToRemove) - } -} - func (c *cache) deleteFromDB(keys []string) []string { if len(keys) == 0 { return keys @@ -119,6 +79,7 @@ func (c *cache) deleteFromDB(keys []string) []string { }) for i := 0; i < errorIndex; i++ { c.objCounters.DecDB() + c.metrics.Evict(StorageTypeDB) storagelog.Write(c.log, storagelog.AddressField(keys[i]), storagelog.StorageTypeField(wcStorageType), @@ -133,7 +94,7 @@ func (c *cache) deleteFromDB(keys []string) []string { return keys[:len(keys)-errorIndex] } -func (c *cache) deleteFromDisk(keys []string) []string { +func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { if len(keys) == 0 { return keys } @@ -147,7 +108,7 @@ func (c *cache) deleteFromDisk(keys []string) []string { continue } - _, err := c.fsTree.Delete(context.TODO(), common.DeletePrm{Address: addr}) + _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err != nil && !errors.As(err, new(apistatus.ObjectNotFound)) { c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) @@ -161,6 +122,7 @@ func (c *cache) deleteFromDisk(keys []string) []string { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), ) + c.metrics.Evict(StorageTypeFSTree) c.objCounters.DecFS() } } diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index bdcc9bbf6..067ff5ae5 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -5,15 +5,14 @@ import ( "os" "sync" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -25,8 +24,8 @@ type Info struct { // Cache represents write-cache for objects. type Cache interface { - Get(ctx context.Context, address oid.Address) (*object.Object, error) - Head(context.Context, oid.Address) (*object.Object, error) + Get(ctx context.Context, address oid.Address) (*objectSDK.Object, error) + Head(context.Context, oid.Address) (*objectSDK.Object, error) // Delete removes object referenced by the given oid.Address from the // Cache. Returns any error encountered that prevented the object to be // removed. @@ -34,7 +33,6 @@ type Cache interface { // Returns apistatus.ObjectNotFound if object is missing in the Cache. // Returns ErrReadOnly if the Cache is currently in the read-only mode. Delete(context.Context, oid.Address) error - Iterate(IterationPrm) error Put(context.Context, common.PutPrm) (common.PutRes, error) SetMode(mode.Mode) error SetLogger(*logger.Logger) @@ -52,18 +50,15 @@ type cache struct { // mtx protects statistics, counters and compressFlags. mtx sync.RWMutex - mode mode.Mode - initialized atomic.Bool - stopInitCh chan struct{} // used to sync initWG initialisation routines and _only_ them - initWG sync.WaitGroup // for initialisation routines only - modeMtx sync.RWMutex + mode mode.Mode + modeMtx sync.RWMutex // compressFlags maps address of a big object to boolean value indicating // whether object should be compressed. compressFlags map[string]struct{} // flushCh is a channel with objects to flush. - flushCh chan *object.Object + flushCh chan *objectSDK.Object // closeCh is close channel, protected by modeMtx. closeCh chan struct{} // wg is a wait group for flush workers. @@ -80,7 +75,7 @@ const wcStorageType = "write-cache" type objectInfo struct { addr string data []byte - obj *object.Object + obj *objectSDK.Object } const ( @@ -96,9 +91,8 @@ var ( // New creates new writecache instance. func New(opts ...Option) Cache { c := &cache{ - flushCh: make(chan *object.Object), - mode: mode.ReadWrite, - stopInitCh: make(chan struct{}), + flushCh: make(chan *objectSDK.Object), + mode: mode.ReadWrite, compressFlags: make(map[string]struct{}), options: options{ @@ -110,6 +104,7 @@ func New(opts ...Option) Cache { maxBatchSize: bbolt.DefaultMaxBatchSize, maxBatchDelay: bbolt.DefaultMaxBatchDelay, openFile: os.OpenFile, + metrics: &metricsStub{}, }, } @@ -117,12 +112,6 @@ func New(opts ...Option) Cache { opts[i](&c.options) } - // Make the LRU cache contain which take approximately 3/4 of the maximum space. - // Assume small and big objects are stored in 50-50 proportion. - c.maxFlushedMarksCount = int(c.maxCacheSize/c.maxObjectSize+c.maxCacheSize/c.smallObjectSize) / 2 * 3 / 4 - // Trigger the removal when the cache is 7/8 full, so that new items can still arrive. - c.maxRemoveBatchSize = c.maxFlushedMarksCount / 8 - return c } @@ -141,43 +130,40 @@ func (c *cache) DumpInfo() Info { func (c *cache) Open(readOnly bool) error { err := c.openStore(readOnly) if err != nil { - return err + return metaerr.Wrap(err) } // Opening after Close is done during maintenance mode, // thus we need to create a channel here. c.closeCh = make(chan struct{}) - return c.initCounters() + return metaerr.Wrap(c.initCounters()) } // Init runs necessary services. func (c *cache) Init() error { - ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.Init") - defer span.End() - - c.initFlushMarks(ctx) + c.metrics.SetMode(c.mode) c.runFlushLoop() return nil } // Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. func (c *cache) Close() error { - // Finish all in-progress operations. - if err := c.setMode(context.TODO(), mode.ReadOnly); err != nil { - return err - } - + // We cannot lock mutex for the whole operation duration + // because it is taken by some background workers, so `wg.Wait()` is done without modeMtx. + c.modeMtx.Lock() if c.closeCh != nil { close(c.closeCh) } + c.mode = mode.DegradedReadOnly // prevent new operations from being processed + c.modeMtx.Unlock() + c.wg.Wait() - if c.closeCh != nil { - c.closeCh = nil - } - c.initialized.Store(false) + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + c.closeCh = nil var err error if c.db != nil { err = c.db.Close() @@ -185,5 +171,6 @@ func (c *cache) Close() error { c.db = nil } } + c.metrics.Close() return nil } diff --git a/pkg/metrics/blobovnizca.go b/pkg/metrics/blobovnizca.go new file mode 100644 index 000000000..9dc3ed572 --- /dev/null +++ b/pkg/metrics/blobovnizca.go @@ -0,0 +1,141 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BlobobvnizcaMetrics interface { + SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) + CloseBlobobvnizcaTree(shardID, path string) + BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) + AddBlobobvnizcaTreePut(shardID, path string, size int) + AddBlobobvnizcaTreeGet(shardID, path string, size int) + + AddTreeSize(shardID, path string, size uint64) + SubTreeSize(shardID, path string, size uint64) + + IncOpenBlobovnizcaCount(shardID, path string) + DecOpenBlobovnizcaCount(shardID, path string) +} + +type blobovnizca struct { + treeMode *shardIDPathModeValue + treeReqDuration *prometheus.HistogramVec + treePut *prometheus.CounterVec + treeGet *prometheus.CounterVec + treeSize *prometheus.GaugeVec + treeOpenCounter *prometheus.GaugeVec +} + +func newBlobovnizca() *blobovnizca { + return &blobovnizca{ + treeMode: newShardIDPathMode(blobovnizaTreeSubSystem, "mode", "Blobovnizca tree mode"), + + treeReqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Blobovnizca tree request process duration", + }, []string{shardIDLabel, pathLabel, successLabel, methodLabel, withStorageIDLabel}), + treePut: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + treeGet: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + treeSize: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "size_bytes", + Help: "Blobovnizca tree size", + }, []string{shardIDLabel, pathLabel}), + treeOpenCounter: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "open_blobovnizca_count", + Help: "Count of opened blobovnizcas of Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + } +} + +func (b *blobovnizca) SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) { + b.treeMode.SetMode(shardID, path, modeFromBool(readOnly)) +} + +func (b *blobovnizca) CloseBlobobvnizcaTree(shardID, path string) { + b.treeMode.SetMode(shardID, path, closedMode) + b.treeReqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.treeGet.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.treePut.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (b *blobovnizca) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) { + b.treeReqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + withStorageIDLabel: withStorageID.String(), + }).Observe(d.Seconds()) +} + +func (b *blobovnizca) AddBlobobvnizcaTreePut(shardID, path string, size int) { + b.treePut.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) AddBlobobvnizcaTreeGet(shardID, path string, size int) { + b.treeGet.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) AddTreeSize(shardID, path string, size uint64) { + b.treeSize.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) SubTreeSize(shardID, path string, size uint64) { + b.treeSize.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Sub(float64(size)) +} + +func (b *blobovnizca) IncOpenBlobovnizcaCount(shardID, path string) { + b.treeOpenCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Inc() +} + +func (b *blobovnizca) DecOpenBlobovnizcaCount(shardID, path string) { + b.treeOpenCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Dec() +} diff --git a/pkg/metrics/blobstore.go b/pkg/metrics/blobstore.go new file mode 100644 index 000000000..d9bb3f029 --- /dev/null +++ b/pkg/metrics/blobstore.go @@ -0,0 +1,87 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BlobstoreMetrics interface { + SetMode(shardID string, readOnly bool) + Close(shardID string) + + MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) + AddPut(shardID string, size int) + AddGet(shardID string, size int) +} + +type blobstoreMetrics struct { + mode *shardIDModeValue + reqDuration *prometheus.HistogramVec + put *prometheus.CounterVec + get *prometheus.CounterVec +} + +func newBlobstoreMetrics() *blobstoreMetrics { + return &blobstoreMetrics{ + mode: newShardIDMode(blobstoreSubSystem, "mode", "Blobstore mode value"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Blobstore request process duration", + }, []string{shardIDLabel, successLabel, methodLabel, withStorageIDLabel}), + put: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to Blobstore", + }, []string{shardIDLabel}), + get: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from Blobstore", + }, []string{shardIDLabel}), + } +} + +func (m *blobstoreMetrics) SetMode(shardID string, readOnly bool) { + m.mode.SetMode(shardID, modeFromBool(readOnly)) +} + +func (m *blobstoreMetrics) Close(shardID string) { + m.mode.SetMode(shardID, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) + m.get.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) + m.put.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) +} + +func (m *blobstoreMetrics) MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + methodLabel: method, + withStorageIDLabel: withStorageID.String(), + }).Observe(d.Seconds()) +} + +func (m *blobstoreMetrics) AddPut(shardID string, size int) { + m.put.With(prometheus.Labels{ + shardIDLabel: shardID, + }).Add(float64(size)) +} + +func (m *blobstoreMetrics) AddGet(shardID string, size int) { + m.get.With(prometheus.Labels{ + shardIDLabel: shardID, + }).Add(float64(size)) +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go new file mode 100644 index 000000000..ae3ac7839 --- /dev/null +++ b/pkg/metrics/consts.go @@ -0,0 +1,45 @@ +package metrics + +const ( + namespace = "frostfs_node" + innerRingNamespace = "frostfs_ir" + + fstreeSubSystem = "fstree" + blobstoreSubSystem = "blobstore" + blobovnizaTreeSubSystem = "blobovniza_tree" + metabaseSubSystem = "metabase" + piloramaSubSystem = "pilorama" + engineSubsystem = "engine" + gcSubsystem = "garbage_collector" + innerRingSubsystem = "ir" + morphSubsystem = "morph" + morphCacheSubsystem = "morphcache" + objectSubsystem = "object" + replicatorSubsystem = "replicator" + stateSubsystem = "state" + treeServiceSubsystem = "treeservice" + writeCacheSubsystem = "writecache" + + successLabel = "success" + shardIDLabel = "shard_id" + modeLabel = "mode" + pathLabel = "path" + methodLabel = "method" + withStorageIDLabel = "with_storage_id" + statusLabel = "status" + objectTypeLabel = "object_type" + typeLabel = "type" + notificationTypeLabel = "notification_type" + invokeTypeLabel = "invoke_type" + contractLabel = "contract" + containerIDLabelKey = "cid" + storageLabel = "storage" + operationLabel = "operation" + + readWriteMode = "READ_WRITE" + readOnlyMode = "READ_ONLY" + closedMode = "CLOSED" + + failedToDeleteStatus = "failed_to_delete" + deletedStatus = "deleted" +) diff --git a/pkg/metrics/desc.go b/pkg/metrics/desc.go deleted file mode 100644 index 74d2d4e6e..000000000 --- a/pkg/metrics/desc.go +++ /dev/null @@ -1,71 +0,0 @@ -package metrics - -import ( - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" -) - -type metric[T prometheus.Collector] struct { - value T - desc Description -} - -// Descriptions contains metric description suitable for further processing. -// The only reason for it to exist is `prometheus.Desc` disallowing field access directly. -// https://github.com/prometheus/client_golang/pull/326 -// https://github.com/prometheus/client_golang/issues/516 -// https://github.com/prometheus/client_golang/issues/222 -type Description struct { - Name string `json:"name"` - Help string `json:"help"` - Type string `json:"type"` - ConstantLabels prometheus.Labels `json:"constant_labels,omitempty"` - VariableLabels []string `json:"variable_labels,omitempty"` -} - -func newGauge(opts prometheus.GaugeOpts) metric[prometheus.Gauge] { - return metric[prometheus.Gauge]{ - value: prometheus.NewGauge(opts), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_GAUGE.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - }, - } -} - -func newGaugeVec(opts prometheus.GaugeOpts, labelNames []string) metric[*prometheus.GaugeVec] { - return metric[*prometheus.GaugeVec]{ - value: prometheus.NewGaugeVec(opts, labelNames), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_GAUGE.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - VariableLabels: labelNames, - }, - } -} - -func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] { - return metric[prometheus.Counter]{ - value: prometheus.NewCounter(opts), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_COUNTER.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - }, - } -} - -// DescribeAll returns descriptions for all registered metrics. -func DescribeAll() ([]Description, error) { - registeredDescriptionsMtx.Lock() - defer registeredDescriptionsMtx.Unlock() - - ds := make([]Description, len(registeredDescriptions)) - copy(ds, registeredDescriptions) - return ds, nil -} diff --git a/pkg/metrics/desc_test.go b/pkg/metrics/desc_test.go deleted file mode 100644 index 28b5e2132..000000000 --- a/pkg/metrics/desc_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package metrics - -import ( - "strings" - "testing" - - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/require" -) - -func TestDescribeAll(t *testing.T) { - const ( - namespace = "my_ns" - subsystem = "mysub" - ) - mustRegister(newCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "my_counter", - })) - - labels := []string{"label1", "label2"} - mustRegister(newGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "my_gauge", - }, labels)) - - constLabels := prometheus.Labels{ - "const1": "abc", - "const2": "xyz", - } - mustRegister(newCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "with_const_labels", - ConstLabels: constLabels, - })) - - descriptions, err := DescribeAll() - require.NoError(t, err) - - seen := make(map[string]bool) - for i := range descriptions { - if !strings.HasPrefix(descriptions[i].Name, namespace) { - continue - } - - require.False(t, seen[descriptions[i].Name], "metric %s was seen twice", descriptions[i].Name) - seen[descriptions[i].Name] = true - - switch descriptions[i].Name { - case prometheus.BuildFQName(namespace, subsystem, "my_counter"): - require.True(t, len(descriptions[i].VariableLabels) == 0) - case prometheus.BuildFQName(namespace, subsystem, "my_gauge"): - require.Equal(t, labels, descriptions[i].VariableLabels) - case prometheus.BuildFQName(namespace, subsystem, "with_const_labels"): - require.Equal(t, len(constLabels), len(descriptions[i].ConstantLabels)) - require.Equal(t, constLabels, descriptions[i].ConstantLabels) - default: - require.FailNow(t, "unexpected metric name: %s", descriptions[i].Name) - } - } - require.Equal(t, 3, len(seen), "not all registered metrics were iterated over") -} diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 28fc1e028..23d799e28 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -1,69 +1,60 @@ package metrics import ( - "fmt" - "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) -type ( - engineMetrics struct { - listContainersDuration metric[prometheus.Counter] - estimateContainerSizeDuration metric[prometheus.Counter] - deleteDuration metric[prometheus.Counter] - existsDuration metric[prometheus.Counter] - getDuration metric[prometheus.Counter] - headDuration metric[prometheus.Counter] - inhumeDuration metric[prometheus.Counter] - putDuration metric[prometheus.Counter] - rangeDuration metric[prometheus.Counter] - searchDuration metric[prometheus.Counter] - listObjectsDuration metric[prometheus.Counter] - containerSize metric[*prometheus.GaugeVec] - payloadSize metric[*prometheus.GaugeVec] - } -) +type EngineMetrics interface { + AddMethodDuration(method string, d time.Duration) + AddToContainerSize(cnrID string, size int64) + IncErrorCounter(shardID string) + ClearErrorCounter(shardID string) + DeleteShardMetrics(shardID string) + AddToObjectCounter(shardID, objectType string, delta int) + SetObjectCounter(shardID, objectType string, v uint64) + AddToPayloadCounter(shardID string, size int64) + SetMode(shardID string, mode mode.Mode) -const engineSubsystem = "engine" + WriteCache() WriteCacheMetrics + GC() GCMetrics +} -func newEngineMetrics() engineMetrics { - return engineMetrics{ - listContainersDuration: newEngineMethodDurationCounter("list_containers_"), - estimateContainerSizeDuration: newEngineCounter("estimate_container_size_duration", "Accumulated duration of engine container size estimate operations"), - deleteDuration: newEngineMethodDurationCounter("delete"), - existsDuration: newEngineMethodDurationCounter("exists"), - getDuration: newEngineMethodDurationCounter("get"), - headDuration: newEngineMethodDurationCounter("head"), - inhumeDuration: newEngineMethodDurationCounter("inhume"), - putDuration: newEngineMethodDurationCounter("put"), - rangeDuration: newEngineMethodDurationCounter("range"), - searchDuration: newEngineMethodDurationCounter("search"), - listObjectsDuration: newEngineMethodDurationCounter("list_objects"), - containerSize: newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), - payloadSize: newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), +type engineMetrics struct { + methodDuration *prometheus.HistogramVec + objectCounter *prometheus.GaugeVec + containerSize *prometheus.GaugeVec + payloadSize *prometheus.GaugeVec + errorCounter *prometheus.GaugeVec + mode *shardIDModeValue + + gc *gcMetrics + writeCache *writeCacheMetrics +} + +func newEngineMetrics() *engineMetrics { + return &engineMetrics{ + containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), + payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabel}), + errorCounter: newEngineGaugeVector("errors_total", "Shard's error counter", []string{shardIDLabel}), + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: engineSubsystem, + Name: "request_duration_seconds", + Help: "Duration of Engine requests", + }, []string{methodLabel}), + objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabel, typeLabel}), + gc: newGCMetrics(), + writeCache: newWriteCacheMetrics(), + mode: newShardIDMode(engineSubsystem, "mode_info", "Shard mode"), } } -func newEngineCounter(name, help string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: name, - Help: help, - }) -} - -func newEngineMethodDurationCounter(method string) metric[prometheus.Counter] { - return newEngineCounter( - fmt.Sprintf("%s_duration", method), - fmt.Sprintf("Accumulated duration of engine %s operations", strings.ReplaceAll(method, "_", " ")), - ) -} - -func newEngineGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] { - return newGaugeVec(prometheus.GaugeOpts{ +func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { + return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: name, @@ -71,70 +62,61 @@ func newEngineGaugeVector(name, help string, labels []string) metric[*prometheus }, labels) } -func (m engineMetrics) register() { - mustRegister(m.listContainersDuration) - mustRegister(m.estimateContainerSizeDuration) - mustRegister(m.deleteDuration) - mustRegister(m.existsDuration) - mustRegister(m.getDuration) - mustRegister(m.headDuration) - mustRegister(m.inhumeDuration) - mustRegister(m.putDuration) - mustRegister(m.rangeDuration) - mustRegister(m.searchDuration) - mustRegister(m.listObjectsDuration) - mustRegister(m.containerSize) - mustRegister(m.payloadSize) +func (m *engineMetrics) AddMethodDuration(method string, d time.Duration) { + m.methodDuration.With(prometheus.Labels{ + methodLabel: method, + }).Observe(d.Seconds()) } -func (m engineMetrics) AddListContainersDuration(d time.Duration) { - m.listObjectsDuration.value.Add(float64(d)) +func (m *engineMetrics) AddToContainerSize(cnrID string, size int64) { + m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) } -func (m engineMetrics) AddEstimateContainerSizeDuration(d time.Duration) { - m.estimateContainerSizeDuration.value.Add(float64(d)) +func (m *engineMetrics) AddToPayloadCounter(shardID string, size int64) { + m.payloadSize.With(prometheus.Labels{shardIDLabel: shardID}).Add(float64(size)) } -func (m engineMetrics) AddDeleteDuration(d time.Duration) { - m.deleteDuration.value.Add(float64(d)) +func (m *engineMetrics) IncErrorCounter(shardID string) { + m.errorCounter.With(prometheus.Labels{shardIDLabel: shardID}).Inc() } -func (m engineMetrics) AddExistsDuration(d time.Duration) { - m.existsDuration.value.Add(float64(d)) +func (m *engineMetrics) ClearErrorCounter(shardID string) { + m.errorCounter.With(prometheus.Labels{shardIDLabel: shardID}).Set(0) } -func (m engineMetrics) AddGetDuration(d time.Duration) { - m.getDuration.value.Add(float64(d)) +func (m *engineMetrics) DeleteShardMetrics(shardID string) { + m.errorCounter.Delete(prometheus.Labels{shardIDLabel: shardID}) + m.payloadSize.Delete(prometheus.Labels{shardIDLabel: shardID}) + m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.mode.Delete(shardID) } -func (m engineMetrics) AddHeadDuration(d time.Duration) { - m.headDuration.value.Add(float64(d)) +func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) { + m.objectCounter.With( + prometheus.Labels{ + shardIDLabel: shardID, + typeLabel: objectType, + }, + ).Add(float64(delta)) } -func (m engineMetrics) AddInhumeDuration(d time.Duration) { - m.inhumeDuration.value.Add(float64(d)) +func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) { + m.objectCounter.With( + prometheus.Labels{ + shardIDLabel: shardID, + typeLabel: objectType, + }, + ).Set(float64(v)) } -func (m engineMetrics) AddPutDuration(d time.Duration) { - m.putDuration.value.Add(float64(d)) +func (m *engineMetrics) SetMode(shardID string, mode mode.Mode) { + m.mode.SetMode(shardID, mode.String()) } -func (m engineMetrics) AddRangeDuration(d time.Duration) { - m.rangeDuration.value.Add(float64(d)) +func (m *engineMetrics) WriteCache() WriteCacheMetrics { + return m.writeCache } -func (m engineMetrics) AddSearchDuration(d time.Duration) { - m.searchDuration.value.Add(float64(d)) -} - -func (m engineMetrics) AddListObjectsDuration(d time.Duration) { - m.listObjectsDuration.value.Add(float64(d)) -} - -func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { - m.containerSize.value.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) -} - -func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) { - m.payloadSize.value.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) +func (m *engineMetrics) GC() GCMetrics { + return m.gc } diff --git a/pkg/metrics/fstree.go b/pkg/metrics/fstree.go new file mode 100644 index 000000000..4d4f0693b --- /dev/null +++ b/pkg/metrics/fstree.go @@ -0,0 +1,92 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type FSTreeMetrics interface { + SetMode(shardID, path string, readOnly bool) + Close(shardID, path string) + + MethodDuration(shardID, path string, method string, d time.Duration, success bool) + AddGet(shardID, path string, size int) + AddPut(shardID, path string, size int) +} + +type fstreeMetrics struct { + mode *shardIDPathModeValue + reqDuration *prometheus.HistogramVec + put *prometheus.CounterVec + get *prometheus.CounterVec +} + +func newFSTreeMetrics() *fstreeMetrics { + return &fstreeMetrics{ + mode: newShardIDPathMode(fstreeSubSystem, "mode", "FSTree mode value"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated FSTree request process duration", + }, []string{shardIDLabel, successLabel, pathLabel, methodLabel}), + put: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to FSTree", + }, []string{shardIDLabel, pathLabel}), + get: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from FSTree", + }, []string{shardIDLabel, pathLabel}), + } +} + +func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) { + m.mode.SetMode(shardID, path, modeFromBool(readOnly)) +} + +func (m *fstreeMetrics) Close(shardID, path string) { + m.mode.SetMode(shardID, path, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + m.get.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + m.put.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (m *fstreeMetrics) MethodDuration(shardID, path string, method string, d time.Duration, success bool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} + +func (m *fstreeMetrics) AddGet(shardID, path string, size int) { + m.get.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (m *fstreeMetrics) AddPut(shardID, path string, size int) { + m.put.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go new file mode 100644 index 000000000..53bfef0e5 --- /dev/null +++ b/pkg/metrics/gc.go @@ -0,0 +1,88 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type GCMetrics interface { + AddRunDuration(shardID string, d time.Duration, success bool) + AddDeletedCount(shardID string, deleted, failed uint64) + AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) + AddInhumedObjectCount(shardID string, count uint64, objectType string) +} + +type gcMetrics struct { + runDuration *prometheus.CounterVec + deletedCounter *prometheus.CounterVec + expCollectDuration *prometheus.CounterVec + inhumedCounter *prometheus.CounterVec +} + +func newGCMetrics() *gcMetrics { + return &gcMetrics{ + runDuration: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "delete_duration_seconds", + Help: "The total time of GC runs to delete objects from disk", + }, []string{shardIDLabel, successLabel}), + deletedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "deleted_objects_total", + Help: "Total count of objects GC deleted or failed to delete from disk", + }, []string{shardIDLabel, statusLabel}), + expCollectDuration: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "marking_duration_seconds", + Help: "The total time of GC runs to mark expired objects as removed", + }, []string{shardIDLabel, successLabel, objectTypeLabel}), + inhumedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "marked_for_removal_objects_total", + Help: "Total count of expired objects GC marked to remove", + }, []string{shardIDLabel, objectTypeLabel}), + } +} + +func (m *gcMetrics) AddRunDuration(shardID string, d time.Duration, success bool) { + m.runDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + }).Add(d.Seconds()) +} + +func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { + m.deletedCounter.With( + prometheus.Labels{ + shardIDLabel: shardID, + statusLabel: deletedStatus, + }).Add(float64(deleted)) + m.deletedCounter.With( + prometheus.Labels{ + shardIDLabel: shardID, + statusLabel: failedToDeleteStatus, + }).Add(float64(failed)) +} + +func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) { + m.expCollectDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + objectTypeLabel: objectType, + }).Add(d.Seconds()) +} + +func (m *gcMetrics) AddInhumedObjectCount(shardID string, count uint64, objectType string) { + m.inhumedCounter.With( + prometheus.Labels{ + shardIDLabel: shardID, + objectTypeLabel: objectType, + }).Add(float64(count)) +} diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 05b76f9c9..d93b3c432 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -1,47 +1,69 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "strconv" + "time" -const innerRingSubsystem = "ir" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { - epoch metric[prometheus.Gauge] - health metric[prometheus.Gauge] + epoch prometheus.Gauge + health prometheus.Gauge + eventDuration *prometheus.HistogramVec + morphCacheMetrics *morphCacheMetrics } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. -func NewInnerRingMetrics() InnerRingServiceMetrics { +func NewInnerRingMetrics() *InnerRingServiceMetrics { var ( - epoch = newGauge(prometheus.GaugeOpts{ - Namespace: namespace, + epoch = metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: innerRingNamespace, Subsystem: innerRingSubsystem, Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) - health = newGauge(prometheus.GaugeOpts{ - Namespace: namespace, + health = metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: innerRingNamespace, Subsystem: innerRingSubsystem, Name: "health", Help: "Current inner-ring node state.", }) + eventDuration = metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: innerRingNamespace, + Subsystem: innerRingSubsystem, + Name: "event_duration_seconds", + Help: "Duration of processing of inner-ring events", + }, []string{typeLabel, successLabel}) ) - mustRegister(epoch) - mustRegister(health) - - return InnerRingServiceMetrics{ - epoch: epoch, - health: health, + return &InnerRingServiceMetrics{ + epoch: epoch, + health: health, + eventDuration: eventDuration, + morphCacheMetrics: newMorphCacheMetrics(innerRingNamespace), } } // SetEpoch updates epoch metrics. -func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) { - m.epoch.value.Set(float64(epoch)) +func (m *InnerRingServiceMetrics) SetEpoch(epoch uint64) { + m.epoch.Set(float64(epoch)) } // SetHealth updates health metrics. -func (m InnerRingServiceMetrics) SetHealth(s int32) { - m.health.value.Set(float64(s)) +func (m *InnerRingServiceMetrics) SetHealth(s int32) { + m.health.Set(float64(s)) +} + +func (m *InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { + m.eventDuration.With(prometheus.Labels{ + typeLabel: typ, + successLabel: strconv.FormatBool(success), + }).Observe(d.Seconds()) +} + +func (m *InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { + return m.morphCacheMetrics } diff --git a/pkg/metrics/metabase.go b/pkg/metrics/metabase.go new file mode 100644 index 000000000..640c7f721 --- /dev/null +++ b/pkg/metrics/metabase.go @@ -0,0 +1,54 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type MetabaseMetrics interface { + SetMode(shardID, path string, mode string) + Close(shardID, path string) + + MethodDuration(shardID, path string, method string, d time.Duration, success bool) +} + +func newMetabaseMetrics() *metabaseMetrics { + return &metabaseMetrics{ + mode: newShardIDPathMode(metabaseSubSystem, "mode", "Metabase mode"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: metabaseSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Metabase request process duration", + }, []string{shardIDLabel, successLabel, pathLabel, methodLabel}), + } +} + +type metabaseMetrics struct { + mode *shardIDPathModeValue + reqDuration *prometheus.HistogramVec +} + +func (m *metabaseMetrics) SetMode(shardID, path string, mode string) { + m.mode.SetMode(shardID, path, mode) +} + +func (m *metabaseMetrics) Close(shardID, path string) { + m.mode.SetMode(shardID, path, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (m *metabaseMetrics) MethodDuration(shardID, path string, method string, d time.Duration, success bool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go new file mode 100644 index 000000000..312a6b33d --- /dev/null +++ b/pkg/metrics/mode.go @@ -0,0 +1,83 @@ +package metrics + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type shardIDModeValue struct { + modeValue *prometheus.GaugeVec +} + +func newShardIDMode(subsystem, name, help string) *shardIDModeValue { + return &shardIDModeValue{ + modeValue: metrics.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: name, + Help: help, + }, []string{shardIDLabel, modeLabel}), + } +} + +func (m *shardIDModeValue) SetMode(shardID string, mode string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) + + m.modeValue.With(prometheus.Labels{ + shardIDLabel: shardID, + modeLabel: mode, + }).Set(1) +} + +func (m *shardIDModeValue) Delete(shardID string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) +} + +type shardIDPathModeValue struct { + modeValue *prometheus.GaugeVec +} + +func newShardIDPathMode(subsystem, name, help string) *shardIDPathModeValue { + return &shardIDPathModeValue{ + modeValue: metrics.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: name, + Help: help, + }, []string{shardIDLabel, pathLabel, modeLabel}), + } +} + +func (m *shardIDPathModeValue) SetMode(shardID, path string, mode string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + + m.modeValue.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + modeLabel: mode, + }).Set(1) +} + +func (m *shardIDPathModeValue) Delete(shardID, path string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func modeFromBool(readOnly bool) string { + modeValue := readWriteMode + if readOnly { + modeValue = readOnlyMode + } + return modeValue +} diff --git a/pkg/metrics/morph.go b/pkg/metrics/morph.go new file mode 100644 index 000000000..5215c674b --- /dev/null +++ b/pkg/metrics/morph.go @@ -0,0 +1,73 @@ +package metrics + +import ( + "strconv" + "time" + + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type morphClientMetrics struct { + switchCount prometheus.Counter + lastBlock prometheus.Gauge + notificationCount *prometheus.CounterVec + invokeDuration *prometheus.HistogramVec +} + +func NewMorphClientMetrics() morphmetrics.Register { + return &morphClientMetrics{ + switchCount: metrics.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "switches_total", + Help: "Number of endpoint switches", + }), + lastBlock: metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "last_block", + Help: "Index of the last received block", + }), + notificationCount: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "notifications_total", + Help: "Number of notifications received by notification type", + }, []string{notificationTypeLabel}), + invokeDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "invoke_duration_seconds", + Help: "Cummulative duration of contract invocations", + }, []string{invokeTypeLabel, contractLabel, methodLabel, successLabel}), + } +} + +func (m *morphClientMetrics) IncSwitchCount() { + m.switchCount.Inc() +} + +func (m *morphClientMetrics) SetLastBlock(index uint32) { + m.lastBlock.Set(float64(index)) +} + +func (m *morphClientMetrics) IncNotificationCount(typ string) { + m.notificationCount.With( + prometheus.Labels{ + notificationTypeLabel: typ, + }, + ).Inc() +} + +func (m *morphClientMetrics) ObserveInvoke(typ string, contract string, method string, success bool, d time.Duration) { + m.invokeDuration.With( + prometheus.Labels{ + invokeTypeLabel: typ, + contractLabel: contract, + methodLabel: method, + successLabel: strconv.FormatBool(success), + }, + ).Observe(d.Seconds()) +} diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go new file mode 100644 index 000000000..a4dbbccfc --- /dev/null +++ b/pkg/metrics/morphcache.go @@ -0,0 +1,43 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type MorphCacheMetrics interface { + AddMethodDuration(method string, success bool, d time.Duration) +} + +type morphCacheMetrics struct { + methodDuration *prometheus.HistogramVec +} + +var _ MorphCacheMetrics = (*morphCacheMetrics)(nil) + +func NewNodeMorphCacheMetrics() MorphCacheMetrics { + return newMorphCacheMetrics(namespace) +} + +func newMorphCacheMetrics(ns string) *morphCacheMetrics { + return &morphCacheMetrics{ + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: ns, + Subsystem: morphCacheSubsystem, + Name: "request_duration_seconds", + Help: "Morph cache request process duration", + }, []string{successLabel, methodLabel}), + } +} + +func (m *morphCacheMetrics) AddMethodDuration(method string, success bool, d time.Duration) { + m.methodDuration.With( + prometheus.Labels{ + successLabel: strconv.FormatBool(success), + methodLabel: method, + }, + ).Observe(d.Seconds()) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 0f9c6183d..45d50b5b0 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -1,43 +1,86 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" - -const namespace = "frostfs_node" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) type NodeMetrics struct { - objectServiceMetrics - engineMetrics - stateMetrics - epoch metric[prometheus.Gauge] + engine *engineMetrics + state *stateMetrics + replicator *replicatorMetrics + objectService *objectServiceMetrics + treeService *treeServiceMetrics + epoch prometheus.Gauge + fstree *fstreeMetrics + blobstore *blobstoreMetrics + blobobvnizca *blobovnizca + metabase *metabaseMetrics + pilorama *piloramaMetrics } func NewNodeMetrics() *NodeMetrics { - objectService := newObjectServiceMetrics() - objectService.register() - - engine := newEngineMetrics() - engine.register() - - state := newStateMetrics() - state.register() - - epoch := newGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: innerRingSubsystem, - Name: "epoch", - Help: "Current epoch as seen by inner-ring node.", - }) - mustRegister(epoch) - return &NodeMetrics{ - objectServiceMetrics: objectService, - engineMetrics: engine, - stateMetrics: state, - epoch: epoch, + objectService: newObjectServiceMetrics(), + engine: newEngineMetrics(), + state: newStateMetrics(), + replicator: newReplicatorMetrics(), + treeService: newTreeServiceMetrics(), + epoch: metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: innerRingSubsystem, + Name: "epoch", + Help: "Current epoch as seen by inner-ring node.", + }), + fstree: newFSTreeMetrics(), + blobstore: newBlobstoreMetrics(), + blobobvnizca: newBlobovnizca(), + metabase: newMetabaseMetrics(), + pilorama: newPiloramaMetrics(), } } // SetEpoch updates epoch metric. func (m *NodeMetrics) SetEpoch(epoch uint64) { - m.epoch.value.Set(float64(epoch)) + m.epoch.Set(float64(epoch)) +} + +func (m *NodeMetrics) TreeService() TreeMetricsRegister { + return m.treeService +} + +func (m *NodeMetrics) Replicator() ReplicatorMetrics { + return m.replicator +} + +func (m *NodeMetrics) ObjectService() ObjectServiceMetrics { + return m.objectService +} + +func (m *NodeMetrics) Engine() EngineMetrics { + return m.engine +} + +func (m *NodeMetrics) State() StateMetrics { + return m.state +} + +func (m *NodeMetrics) FSTree() FSTreeMetrics { + return m.fstree +} + +func (m *NodeMetrics) Blobstore() BlobstoreMetrics { + return m.blobstore +} + +func (m *NodeMetrics) BlobobvnizcaTreeMetrics() BlobobvnizcaMetrics { + return m.blobobvnizca +} + +func (m *NodeMetrics) MetabaseMetrics() MetabaseMetrics { + return m.metabase +} + +func (m *NodeMetrics) PiloramaMetrics() PiloramaMetrics { + return m.pilorama } diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index 5ec575749..0ba994ed3 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -1,245 +1,49 @@ package metrics import ( - "fmt" - "strings" + "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) -const objectSubsystem = "object" +type ObjectServiceMetrics interface { + AddRequestDuration(method string, d time.Duration, success bool) + AddPayloadSize(method string, size int) +} -type ( - methodCount struct { - success metric[prometheus.Counter] - total metric[prometheus.Counter] - } +type objectServiceMetrics struct { + methodDuration *prometheus.HistogramVec + payloadCounter *prometheus.CounterVec +} - objectServiceMetrics struct { - getCounter methodCount - putCounter methodCount - headCounter methodCount - searchCounter methodCount - deleteCounter methodCount - rangeCounter methodCount - rangeHashCounter methodCount - - getDuration metric[prometheus.Counter] - putDuration metric[prometheus.Counter] - headDuration metric[prometheus.Counter] - searchDuration metric[prometheus.Counter] - deleteDuration metric[prometheus.Counter] - rangeDuration metric[prometheus.Counter] - rangeHashDuration metric[prometheus.Counter] - - putPayload metric[prometheus.Counter] - getPayload metric[prometheus.Counter] - - shardMetrics metric[*prometheus.GaugeVec] - shardsReadonly metric[*prometheus.GaugeVec] - } -) - -const ( - shardIDLabelKey = "shard" - counterTypeLabelKey = "type" - containerIDLabelKey = "cid" -) - -func newObjectMethodCallCounter(name string) methodCount { - return methodCount{ - success: newCounter(prometheus.CounterOpts{ +func newObjectServiceMetrics() *objectServiceMetrics { + return &objectServiceMetrics{ + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_req_count_success", name), - Help: fmt.Sprintf("The number of successful %s requests processed", name), - }), - total: newCounter(prometheus.CounterOpts{ + Name: "request_duration_seconds", + Help: "Object Service request process duration", + }, []string{methodLabel, successLabel}), + payloadCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_req_count", name), - Help: fmt.Sprintf("Total number of %s requests processed", name), - }), + Name: "request_payload_bytes", + Help: "Object Service request payload", + }, []string{methodLabel}), } } -func (m methodCount) mustRegister() { - mustRegister(m.success) - mustRegister(m.total) +func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool) { + m.methodDuration.With(prometheus.Labels{ + methodLabel: method, + successLabel: strconv.FormatBool(success), + }).Observe(d.Seconds()) } -func (m methodCount) Inc(success bool) { - m.total.value.Inc() - if success { - m.success.value.Inc() - } -} - -func newObjectServiceMetrics() objectServiceMetrics { - return objectServiceMetrics{ - getCounter: newObjectMethodCallCounter("get"), - putCounter: newObjectMethodCallCounter("put"), - headCounter: newObjectMethodCallCounter("head"), - searchCounter: newObjectMethodCallCounter("search"), - deleteCounter: newObjectMethodCallCounter("delete"), - rangeCounter: newObjectMethodCallCounter("range"), - rangeHashCounter: newObjectMethodCallCounter("range_hash"), - getDuration: newObjectMethodDurationCounter("get"), - putDuration: newObjectMethodDurationCounter("put"), - headDuration: newObjectMethodDurationCounter("head"), - searchDuration: newObjectMethodDurationCounter("search"), - deleteDuration: newObjectMethodDurationCounter("delete"), - rangeDuration: newObjectMethodDurationCounter("range"), - rangeHashDuration: newObjectMethodDurationCounter("range_hash"), - putPayload: newObjectMethodPayloadCounter("put"), - getPayload: newObjectMethodPayloadCounter("get"), - shardMetrics: newObjectGaugeVector("counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), - shardsReadonly: newObjectGaugeVector("readonly", "Shard state", []string{shardIDLabelKey}), - } -} - -func newObjectMethodPayloadCounter(method string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_payload", method), - Help: fmt.Sprintf("Accumulated payload size at object %s method", strings.ReplaceAll(method, "_", " ")), - }) -} - -func newObjectMethodDurationCounter(method string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_req_duration", method), - Help: fmt.Sprintf("Accumulated %s request process duration", strings.ReplaceAll(method, "_", " ")), - }) -} - -func newObjectGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] { - return newGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: name, - Help: help, - }, labels) -} - -func (m objectServiceMetrics) register() { - m.getCounter.mustRegister() - m.putCounter.mustRegister() - m.headCounter.mustRegister() - m.searchCounter.mustRegister() - m.deleteCounter.mustRegister() - m.rangeCounter.mustRegister() - m.rangeHashCounter.mustRegister() - - mustRegister(m.getDuration) - mustRegister(m.putDuration) - mustRegister(m.headDuration) - mustRegister(m.searchDuration) - mustRegister(m.deleteDuration) - mustRegister(m.rangeDuration) - mustRegister(m.rangeHashDuration) - - mustRegister(m.putPayload) - mustRegister(m.getPayload) - - mustRegister(m.shardMetrics) - mustRegister(m.shardsReadonly) -} - -func (m objectServiceMetrics) IncGetReqCounter(success bool) { - m.getCounter.Inc(success) -} - -func (m objectServiceMetrics) IncPutReqCounter(success bool) { - m.putCounter.Inc(success) -} - -func (m objectServiceMetrics) IncHeadReqCounter(success bool) { - m.headCounter.Inc(success) -} - -func (m objectServiceMetrics) IncSearchReqCounter(success bool) { - m.searchCounter.Inc(success) -} - -func (m objectServiceMetrics) IncDeleteReqCounter(success bool) { - m.deleteCounter.Inc(success) -} - -func (m objectServiceMetrics) IncRangeReqCounter(success bool) { - m.rangeCounter.Inc(success) -} - -func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) { - m.rangeHashCounter.Inc(success) -} - -func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) { - m.getDuration.value.Add(float64(d)) -} - -func (m objectServiceMetrics) AddPutReqDuration(d time.Duration) { - m.putDuration.value.Add(float64(d)) -} - -func (m objectServiceMetrics) AddHeadReqDuration(d time.Duration) { - m.headDuration.value.Add(float64(d)) -} - -func (m objectServiceMetrics) AddSearchReqDuration(d time.Duration) { - m.searchDuration.value.Add(float64(d)) -} - -func (m objectServiceMetrics) AddDeleteReqDuration(d time.Duration) { - m.deleteDuration.value.Add(float64(d)) -} - -func (m objectServiceMetrics) AddRangeReqDuration(d time.Duration) { - m.rangeDuration.value.Add(float64(d)) -} - -func (m objectServiceMetrics) AddRangeHashReqDuration(d time.Duration) { - m.rangeHashDuration.value.Add(float64(d)) -} - -func (m objectServiceMetrics) AddPutPayload(ln int) { - m.putPayload.value.Add(float64(ln)) -} - -func (m objectServiceMetrics) AddGetPayload(ln int) { - m.getPayload.value.Add(float64(ln)) -} - -func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, delta int) { - m.shardMetrics.value.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - counterTypeLabelKey: objectType, - }, - ).Add(float64(delta)) -} - -func (m objectServiceMetrics) SetObjectCounter(shardID, objectType string, v uint64) { - m.shardMetrics.value.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - counterTypeLabelKey: objectType, - }, - ).Set(float64(v)) -} - -func (m objectServiceMetrics) SetReadonly(shardID string, readonly bool) { - var flag float64 - if readonly { - flag = 1 - } - m.shardsReadonly.value.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - }, - ).Set(flag) +func (m *objectServiceMetrics) AddPayloadSize(method string, size int) { + m.payloadCounter.With(prometheus.Labels{ + methodLabel: method, + }).Add(float64(size)) } diff --git a/pkg/metrics/pilorama.go b/pkg/metrics/pilorama.go new file mode 100644 index 000000000..41672a4b5 --- /dev/null +++ b/pkg/metrics/pilorama.go @@ -0,0 +1,53 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type PiloramaMetrics interface { + SetMode(shardID string, m mode.Mode) + Close(shardID string) + + AddMethodDuration(shardID string, method string, d time.Duration, success bool) +} + +func newPiloramaMetrics() *piloramaMetrics { + return &piloramaMetrics{ + mode: newShardIDMode(piloramaSubSystem, "mode", "Pilorama mode"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: piloramaSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Pilorama request process duration", + }, []string{shardIDLabel, successLabel, methodLabel}), + } +} + +type piloramaMetrics struct { + mode *shardIDModeValue + reqDuration *prometheus.HistogramVec +} + +func (m *piloramaMetrics) SetMode(shardID string, mode mode.Mode) { + m.mode.SetMode(shardID, mode.String()) +} + +func (m *piloramaMetrics) AddMethodDuration(shardID string, method string, d time.Duration, success bool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} + +func (m *piloramaMetrics) Close(shardID string) { + m.mode.SetMode(shardID, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) +} diff --git a/pkg/metrics/registry.go b/pkg/metrics/registry.go deleted file mode 100644 index eef613d04..000000000 --- a/pkg/metrics/registry.go +++ /dev/null @@ -1,42 +0,0 @@ -package metrics - -import ( - "net/http" - "sync" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -// Handler returns an http.Handler for the local registry. -func Handler() http.Handler { - promhttp.Handler() - return promhttp.InstrumentMetricHandler( - registry, - promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) -} - -var ( - registry = prometheus.NewRegistry() - // registeredDescriptionsMtx protects collectors slice. - // It should not be acessed concurrently, but we can easily forget this in future, thus this mutex. - registeredDescriptionsMtx sync.Mutex - registeredDescriptions []Description -) - -func init() { - registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) - registry.MustRegister(collectors.NewGoCollector()) -} - -func mustRegister[T prometheus.Collector](cs ...metric[T]) { - for i := range cs { - registry.MustRegister(cs[i].value) - } - registeredDescriptionsMtx.Lock() - for i := range cs { - registeredDescriptions = append(registeredDescriptions, cs[i].desc) - } - registeredDescriptionsMtx.Unlock() -} diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go new file mode 100644 index 000000000..a1519ac95 --- /dev/null +++ b/pkg/metrics/replicator.go @@ -0,0 +1,63 @@ +package metrics + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +//TODO + +type ReplicatorMetrics interface { + IncInFlightRequest() + DecInFlightRequest() + IncProcessedObjects() + AddPayloadSize(size int64) +} + +type replicatorMetrics struct { + inFlightRequests prometheus.Gauge + processedObjects prometheus.Counter + totalReplicatedPayloadSize prometheus.Counter +} + +func (m *replicatorMetrics) IncInFlightRequest() { + m.inFlightRequests.Inc() +} + +func (m *replicatorMetrics) DecInFlightRequest() { + m.inFlightRequests.Dec() +} + +func (m *replicatorMetrics) IncProcessedObjects() { + m.processedObjects.Inc() +} + +func (m *replicatorMetrics) AddPayloadSize(size int64) { + m.totalReplicatedPayloadSize.Add(float64(size)) +} + +func newReplicatorMetrics() *replicatorMetrics { + return &replicatorMetrics{ + inFlightRequests: newReplicatorGauge("in_flight_requests_total", "Number of in-flight requests"), + processedObjects: newReplicatorCounter("processed_objects_total", "Number of objects processed since the node startup"), + totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size_bytes", "Total size of payloads replicated"), + } +} + +func newReplicatorCounter(name, help string) prometheus.Counter { + return metrics.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: replicatorSubsystem, + Name: name, + Help: help, + }) +} + +func newReplicatorGauge(name, help string) prometheus.Gauge { + return metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: replicatorSubsystem, + Name: name, + Help: help, + }) +} diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index dce0402cd..243f648e5 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -1,16 +1,21 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) -const stateSubsystem = "state" - -type stateMetrics struct { - healthCheck metric[prometheus.Gauge] +type StateMetrics interface { + SetHealth(s int32) } -func newStateMetrics() stateMetrics { - return stateMetrics{ - healthCheck: newGauge(prometheus.GaugeOpts{ +type stateMetrics struct { + healthCheck prometheus.Gauge +} + +func newStateMetrics() *stateMetrics { + return &stateMetrics{ + healthCheck: metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: stateSubsystem, Name: "health", @@ -19,10 +24,6 @@ func newStateMetrics() stateMetrics { } } -func (m stateMetrics) register() { - mustRegister(m.healthCheck) -} - -func (m stateMetrics) SetHealth(s int32) { - m.healthCheck.value.Set(float64(s)) +func (m *stateMetrics) SetHealth(s int32) { + m.healthCheck.Set(float64(s)) } diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go new file mode 100644 index 000000000..6702aa83c --- /dev/null +++ b/pkg/metrics/treeservice.go @@ -0,0 +1,64 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type TreeMetricsRegister interface { + AddReplicateTaskDuration(time.Duration, bool) + AddReplicateWaitDuration(time.Duration, bool) + AddSyncDuration(time.Duration, bool) +} + +type treeServiceMetrics struct { + replicateTaskDuration *prometheus.HistogramVec + replicateWaitDuration *prometheus.HistogramVec + syncOpDuration *prometheus.HistogramVec +} + +var _ TreeMetricsRegister = (*treeServiceMetrics)(nil) + +func newTreeServiceMetrics() *treeServiceMetrics { + return &treeServiceMetrics{ + replicateTaskDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: treeServiceSubsystem, + Name: "replicate_task_duration_seconds", + Help: "Duration of individual replication tasks executed as part of replication loops", + }, []string{successLabel}), + replicateWaitDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: treeServiceSubsystem, + Name: "replicate_wait_duration_seconds", + Help: "Duration of overall waiting time for replication loops", + }, []string{successLabel}), + syncOpDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: treeServiceSubsystem, + Name: "sync_duration_seconds", + Help: "Duration of synchronization operations", + }, []string{successLabel}), + } +} + +func (m *treeServiceMetrics) AddReplicateTaskDuration(d time.Duration, success bool) { + m.replicateTaskDuration.With(prometheus.Labels{ + successLabel: strconv.FormatBool(success), + }).Observe(d.Seconds()) +} + +func (m *treeServiceMetrics) AddReplicateWaitDuration(d time.Duration, success bool) { + m.replicateWaitDuration.With(prometheus.Labels{ + successLabel: strconv.FormatBool(success), + }).Observe(d.Seconds()) +} + +func (m *treeServiceMetrics) AddSyncDuration(d time.Duration, success bool) { + m.syncOpDuration.With(prometheus.Labels{ + successLabel: strconv.FormatBool(success), + }).Observe(d.Seconds()) +} diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go new file mode 100644 index 000000000..6a76248bf --- /dev/null +++ b/pkg/metrics/types.go @@ -0,0 +1,17 @@ +package metrics + +import ( + "strconv" +) + +type NullBool struct { + Bool bool + Valid bool // Valid is true if Bool is not NULL +} + +func (v NullBool) String() string { + if !v.Valid { + return "" + } + return strconv.FormatBool(v.Bool) +} diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go new file mode 100644 index 000000000..7e6083a49 --- /dev/null +++ b/pkg/metrics/writecache.go @@ -0,0 +1,124 @@ +package metrics + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type WriteCacheMetrics interface { + AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) + + IncActualCount(shardID string, storageType string) + DecActualCount(shardID string, storageType string) + SetActualCount(shardID string, count uint64, storageType string) + + SetEstimateSize(shardID string, size uint64, storageType string) + SetMode(shardID string, mode string) + + IncOperationCounter(shardID string, operation string, success NullBool, storageType string) + + Close(shardID string) +} + +type writeCacheMetrics struct { + methodDuration *prometheus.HistogramVec + operationCounter *prometheus.CounterVec + + actualCount *prometheus.GaugeVec + + estimatedSize *prometheus.GaugeVec + + mode *shardIDModeValue +} + +func newWriteCacheMetrics() *writeCacheMetrics { + return &writeCacheMetrics{ + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: writeCacheSubsystem, + Name: "request_duration_seconds", + Help: "Writecache request process duration", + }, []string{shardIDLabel, successLabel, storageLabel, methodLabel}), + operationCounter: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: writeCacheSubsystem, + Name: "operations_total", + Help: "The number of writecache operations processed", + }, []string{shardIDLabel, storageLabel, successLabel, operationLabel}), + actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{shardIDLabel, storageLabel}), + estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{shardIDLabel, storageLabel}), + mode: newShardIDMode(writeCacheSubsystem, "mode_info", "Writecache mode value"), + } +} + +func (m *writeCacheMetrics) AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) { + m.methodDuration.With( + prometheus.Labels{ + shardIDLabel: shardID, + successLabel: fmt.Sprintf("%v", success), + storageLabel: storageType, + methodLabel: method, + }, + ).Observe(d.Seconds()) +} + +func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { + m.actualCount.With(prometheus.Labels{ + shardIDLabel: shardID, + storageLabel: storageType, + }).Inc() +} + +func (m *writeCacheMetrics) DecActualCount(shardID string, storageType string) { + m.actualCount.With(prometheus.Labels{ + shardIDLabel: shardID, + storageLabel: storageType, + }).Dec() +} + +func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) { + m.actualCount.With(prometheus.Labels{ + shardIDLabel: shardID, + storageLabel: storageType, + }).Set(float64(count)) +} + +func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storageType string) { + m.estimatedSize.With(prometheus.Labels{ + shardIDLabel: shardID, + storageLabel: storageType, + }).Set(float64(size)) +} + +func (m *writeCacheMetrics) SetMode(shardID string, mode string) { + m.mode.SetMode(shardID, mode) +} + +func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) { + m.operationCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + storageLabel: storageType, + operationLabel: operation, + successLabel: success.String(), + }).Inc() +} + +func (m *writeCacheMetrics) Close(shardID string) { + m.mode.Delete(shardID) + m.methodDuration.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.operationCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.actualCount.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.estimatedSize.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) +} + +func newWCGaugeVec(name, help string, labels []string) *prometheus.GaugeVec { + return metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: writeCacheSubsystem, + Name: name, + Help: help, + }, labels) +} diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 1c33fa5e0..606f3bd66 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -6,9 +6,12 @@ import ( "fmt" "math/big" "sync" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -21,13 +24,11 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/rpcclient/rolemgmt" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" - sc "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -48,7 +49,8 @@ import ( type Client struct { cache cache - logger *logger.Logger // logging component + logger *logger.Logger // logging component + metrics morphmetrics.Register client *rpcclient.WSClient // neo-go websocket client rpcActor *actor.Actor // neo-go RPC actor @@ -67,10 +69,7 @@ type Client struct { // switchLock protects endpoints, inactive, and subscription-related fields. // It is taken exclusively during endpoint switch and locked in shared mode // on every normal call. - switchLock *sync.RWMutex - - notifications chan rpcclient.Notification - subsInfo // protected with switchLock + switchLock sync.RWMutex // channel for internal stop closeChan chan struct{} @@ -87,14 +86,16 @@ type Client struct { } type cache struct { - m *sync.RWMutex + m sync.RWMutex nnsHash *util.Uint160 gKey *keys.PublicKey txHeights *lru.Cache[util.Uint256, uint32] + + metrics metrics.MorphCacheMetrics } -func (c cache) nns() *util.Uint160 { +func (c *cache) nns() *util.Uint160 { c.m.RLock() defer c.m.RUnlock() @@ -108,7 +109,7 @@ func (c *cache) setNNSHash(nnsHash util.Uint160) { c.nnsHash = &nnsHash } -func (c cache) groupKey() *keys.PublicKey { +func (c *cache) groupKey() *keys.PublicKey { c.m.RLock() defer c.m.RUnlock() @@ -156,8 +157,6 @@ func (e *notHaltStateError) Error() string { ) } -var errEmptyInvocationScript = errors.New("got empty invocation script from neo node") - // implementation of error interface for FrostFS-specific errors. type frostfsError struct { err error @@ -175,6 +174,12 @@ func wrapFrostFSError(err error) error { // Invoke invokes contract method by sending transaction into blockchain. // Supported args types: int64, string, util.Uint160, []byte and bool. func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("Invoke", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -192,6 +197,7 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, zap.Uint32("vub", vub), zap.Stringer("tx_hash", txHash.Reverse())) + success = true return nil } @@ -199,6 +205,12 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, // If cb returns an error, the session is closed and this error is returned as-is. // If the remove neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned. func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, contract util.Uint160, method string, args ...interface{}) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("TestInvokeIterator", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -231,12 +243,20 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, contract util } items, err = c.rpcActor.TraverseIterator(sid, &r, 0) } + + success = err == nil return err } // TestInvoke invokes contract method locally in neo-go node. This method should // be used to read data from smart-contract. func (c *Client) TestInvoke(contract util.Uint160, method string, args ...any) (res []stackitem.Item, err error) { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("TestInvoke", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -253,6 +273,7 @@ func (c *Client) TestInvoke(contract util.Uint160, method string, args ...any) ( return nil, wrapFrostFSError(¬HaltStateError{state: val.State, exception: val.FaultException}) } + success = true return val.Stack, nil } @@ -449,64 +470,6 @@ func (c *Client) roleList(r noderoles.Role) (keys.PublicKeys, error) { return c.rolemgmt.GetDesignatedByRole(r, height) } -// tries to resolve sc.Parameter from the arg. -// -// Wraps any error to frostfsError. -func toStackParameter(value any) (sc.Parameter, error) { - var res = sc.Parameter{ - Value: value, - } - - switch v := value.(type) { - case []byte: - res.Type = sc.ByteArrayType - case int: - res.Type = sc.IntegerType - res.Value = big.NewInt(int64(v)) - case int64: - res.Type = sc.IntegerType - res.Value = big.NewInt(v) - case uint64: - res.Type = sc.IntegerType - res.Value = new(big.Int).SetUint64(v) - case [][]byte: - arr := make([]sc.Parameter, 0, len(v)) - for i := range v { - elem, err := toStackParameter(v[i]) - if err != nil { - return res, err - } - - arr = append(arr, elem) - } - - res.Type = sc.ArrayType - res.Value = arr - case string: - res.Type = sc.StringType - case util.Uint160: - res.Type = sc.ByteArrayType - res.Value = v.BytesBE() - case noderoles.Role: - res.Type = sc.IntegerType - res.Value = big.NewInt(int64(v)) - case keys.PublicKeys: - arr := make([][]byte, 0, len(v)) - for i := range v { - arr = append(arr, v[i].Bytes()) - } - - return toStackParameter(arr) - case bool: - res.Type = sc.BoolType - res.Value = v - default: - return res, wrapFrostFSError(fmt.Errorf("chain/client: unsupported parameter %v", value)) - } - - return res, nil -} - // MagicNumber returns the magic number of the network // to which the underlying RPC node client is connected. func (c *Client) MagicNumber() (uint64, error) { @@ -566,26 +529,15 @@ func (c *Client) IsValidScript(script []byte, signers []transaction.Signer) (val // NotificationChannel returns channel than receives subscribed // notification from the connected RPC node. -// Channel is closed when connection to the RPC node has been -// lost without the possibility of recovery. +// Channel is closed when connection to the RPC node is lost. func (c *Client) NotificationChannel() <-chan rpcclient.Notification { - return c.notifications + c.switchLock.RLock() + defer c.switchLock.RUnlock() + return c.client.Notifications //lint:ignore SA1019 waits for neo-go v0.102.0 https://github.com/nspcc-dev/neo-go/pull/2980 } -// inactiveMode switches Client to an inactive mode: -// - notification channel is closed; -// - all the new RPC request would return ErrConnectionLost; -// - inactiveModeCb is called if not nil. -func (c *Client) inactiveMode() { - c.switchLock.Lock() - defer c.switchLock.Unlock() - - close(c.notifications) - c.inactive = true - - if c.cfg.inactiveModeCb != nil { - c.cfg.inactiveModeCb() - } +func (c *Client) Metrics() morphmetrics.Register { + return c.metrics } func (c *Client) setActor(act *actor.Actor) { diff --git a/pkg/morph/client/client_test.go b/pkg/morph/client/client_test.go deleted file mode 100644 index a448c2cf4..000000000 --- a/pkg/morph/client/client_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package client - -import ( - "math/big" - "testing" - - sc "github.com/nspcc-dev/neo-go/pkg/smartcontract" - "github.com/stretchr/testify/require" -) - -func TestToStackParameter(t *testing.T) { - items := []struct { - value any - expType sc.ParamType - expVal any - }{ - { - value: []byte{1, 2, 3}, - expType: sc.ByteArrayType, - }, - { - value: int64(100), - expType: sc.IntegerType, - expVal: big.NewInt(100), - }, - { - value: uint64(100), - expType: sc.IntegerType, - expVal: big.NewInt(100), - }, - { - value: "hello world", - expType: sc.StringType, - }, - { - value: false, - expType: sc.BoolType, - }, - { - value: true, - expType: sc.BoolType, - }, - } - - for _, item := range items { - t.Run(item.expType.String()+" to stack parameter", func(t *testing.T) { - res, err := toStackParameter(item.value) - require.NoError(t, err) - require.Equal(t, item.expType, res.Type) - if item.expVal != nil { - require.Equal(t, item.expVal, res.Value) - } else { - require.Equal(t, item.value, res.Value) - } - }) - } -} diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 4232b349d..e7e1bbca9 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -4,17 +4,15 @@ import ( "context" "errors" "fmt" - "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" - "github.com/nspcc-dev/neo-go/pkg/core/block" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/util" @@ -35,6 +33,8 @@ type cfg struct { logger *logger.Logger // logging component + metrics morphmetrics.Register + waitInterval time.Duration signer *transaction.Signer @@ -46,6 +46,8 @@ type cfg struct { inactiveModeCb Callback switchInterval time.Duration + + morphCacheMetrics metrics.MorphCacheMetrics } const ( @@ -61,10 +63,12 @@ func defaultConfig() *cfg { return &cfg{ dialTimeout: defaultDialTimeout, logger: &logger.Logger{Logger: zap.L()}, + metrics: morphmetrics.NoopRegister{}, waitInterval: defaultWaitInterval, signer: &transaction.Signer{ Scopes: transaction.Global, }, + morphCacheMetrics: &morphmetrics.NoopMorphCacheMetrics{}, } } @@ -81,6 +85,7 @@ func defaultConfig() *cfg { // - signer with the global scope; // - wait interval: 500ms; // - logger: &logger.Logger{Logger: zap.L()}. +// - metrics: metrics.NoopRegister // // If desired option satisfies the default value, it can be omitted. // If multiple options of the same config value are supplied, @@ -108,20 +113,12 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli := &Client{ - cache: newClientCache(), - logger: cfg.logger, - acc: acc, - accAddr: accAddr, - cfg: *cfg, - switchLock: &sync.RWMutex{}, - notifications: make(chan rpcclient.Notification), - subsInfo: subsInfo{ - blockRcv: make(chan *block.Block), - notificationRcv: make(chan *state.ContainedNotificationEvent), - notaryReqRcv: make(chan *result.NotaryRequestEvent), - subscribedEvents: make(map[util.Uint160]string), - subscribedNotaryEvents: make(map[util.Uint160]string), - }, + cache: newClientCache(cfg.morphCacheMetrics), + logger: cfg.logger, + metrics: cfg.metrics, + acc: acc, + accAddr: accAddr, + cfg: *cfg, closeChan: make(chan struct{}), } @@ -162,14 +159,16 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli.setActor(act) - go cli.notificationLoop(ctx) + go cli.closeWaiter(ctx) return cli, nil } func (c *Client) newCli(ctx context.Context, endpoint string) (*rpcclient.WSClient, *actor.Actor, error) { - cli, err := rpcclient.NewWS(ctx, endpoint, rpcclient.Options{ - DialTimeout: c.cfg.dialTimeout, + cli, err := rpcclient.NewWS(ctx, endpoint, rpcclient.WSOptions{ + Options: rpcclient.Options{ + DialTimeout: c.cfg.dialTimeout, + }, }) if err != nil { return nil, nil, fmt.Errorf("WS client creation: %w", err) @@ -206,11 +205,11 @@ func newActor(ws *rpcclient.WSClient, acc *wallet.Account, cfg cfg) (*actor.Acto }}) } -func newClientCache() cache { +func newClientCache(morphCacheMetrics metrics.MorphCacheMetrics) cache { c, _ := lru.New[util.Uint256, uint32](100) // returns error only if size is negative return cache{ - m: &sync.RWMutex{}, txHeights: c, + metrics: morphCacheMetrics, } } @@ -243,6 +242,20 @@ func WithLogger(logger *logger.Logger) Option { } } +// WithMetrics returns a client constructor option +// that specifies the component for reporting metrics. +// +// Ignores nil value. +// +// If option not provided, NoopMetrics is used. +func WithMetrics(metrics morphmetrics.Register) Option { + return func(c *cfg) { + if metrics != nil { + c.metrics = metrics + } + } +} + // WithSigner returns a client constructor option // that specifies the signer and the scope of the transaction. // @@ -294,3 +307,9 @@ func WithSwitchInterval(i time.Duration) Option { c.switchInterval = i } } + +func WithMorphCacheMetrics(morphCacheMetrics metrics.MorphCacheMetrics) Option { + return func(c *cfg) { + c.morphCacheMetrics = morphCacheMetrics + } +} diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index 85d742328..2b5996cd7 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -56,10 +56,6 @@ func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, opts[i](o) } - if o.feePutNamedSet { - o.staticOpts = append(o.staticOpts, client.WithCustomFee(putNamedMethod, o.feePutNamed)) - } - sc, err := client.NewStatic(cli, contract, fee, o.staticOpts...) if err != nil { return nil, fmt.Errorf("can't create container static client: %w", err) @@ -83,9 +79,6 @@ func (c Client) ContractAddress() util.Uint160 { type Option func(*opts) type opts struct { - feePutNamedSet bool - feePutNamed fixedn.Fixed8 - staticOpts []client.StaticClientOption } @@ -111,11 +104,3 @@ func AsAlphabet() Option { o.staticOpts = append(o.staticOpts, client.AsAlphabet()) } } - -// WithCustomFeeForNamedPut returns option to specify custom fee for each Put operation with named container. -func WithCustomFeeForNamedPut(fee fixedn.Fixed8) Option { - return func(o *opts) { - o.feePutNamed = fee - o.feePutNamedSet = true - } -} diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index c9105a3ca..5bc8fc188 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -14,14 +14,15 @@ import ( // Returns error if container ID is nil. func Delete(c *Client, witness core.RemovalWitness) error { binCnr := make([]byte, sha256.Size) - witness.ContainerID().Encode(binCnr) + witness.ContainerID.Encode(binCnr) var prm DeletePrm prm.SetCID(binCnr) - prm.SetSignature(witness.Signature()) + prm.SetSignature(witness.Signature.GetSign()) + prm.SetKey(witness.Signature.GetKey()) - if tok := witness.SessionToken(); tok != nil { + if tok := witness.SessionToken; tok != nil { prm.SetToken(tok.Marshal()) } @@ -33,6 +34,7 @@ type DeletePrm struct { cnr []byte signature []byte token []byte + key []byte client.InvokePrmOptional } @@ -52,6 +54,11 @@ func (d *DeletePrm) SetToken(token []byte) { d.token = token } +// SetKey sets public key. +func (d *DeletePrm) SetKey(key []byte) { + d.key = key +} + // Delete removes the container from FrostFS system // through Container contract call. // @@ -66,7 +73,7 @@ func (c *Client) Delete(p DeletePrm) error { prm := client.InvokePrm{} prm.SetMethod(deleteMethod) - prm.SetArgs(p.cnr, p.signature, p.token) + prm.SetArgs(p.cnr, p.signature, p.key, p.token) prm.InvokePrmOptional = p.InvokePrmOptional err := c.client.Invoke(prm) diff --git a/pkg/morph/client/container/eacl.go b/pkg/morph/client/container/eacl.go index 56c36c179..54c83737d 100644 --- a/pkg/morph/client/container/eacl.go +++ b/pkg/morph/client/container/eacl.go @@ -86,7 +86,7 @@ func (c *Client) GetEACL(cnr cid.ID) (*container.EACL, error) { } } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature sigV2.SetKey(pub) sigV2.SetSign(sig) diff --git a/pkg/morph/client/container/eacl_set.go b/pkg/morph/client/container/eacl_set.go index 86eae4c2b..2d2ffb456 100644 --- a/pkg/morph/client/container/eacl_set.go +++ b/pkg/morph/client/container/eacl_set.go @@ -31,7 +31,7 @@ func PutEACL(c *Client, eaclInfo containercore.EACL) error { prm.SetToken(eaclInfo.Session.Marshal()) } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature eaclInfo.Signature.WriteToV2(&sigV2) diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index 009b22f3c..0513eea4b 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -105,7 +105,7 @@ func (c *Client) Get(cid []byte) (*containercore.Container, error) { } } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature sigV2.SetKey(pub) sigV2.SetSign(sigBytes) diff --git a/pkg/morph/client/container/put.go b/pkg/morph/client/container/put.go index 2c97446c6..5c23eb36d 100644 --- a/pkg/morph/client/container/put.go +++ b/pkg/morph/client/container/put.go @@ -28,7 +28,7 @@ func Put(c *Client, cnr containercore.Container) (*cid.ID, error) { prm.SetToken(cnr.Session.Marshal()) } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature cnr.Signature.WriteToV2(&sigV2) diff --git a/pkg/morph/client/fee.go b/pkg/morph/client/fee.go deleted file mode 100644 index 8a38c4f55..000000000 --- a/pkg/morph/client/fee.go +++ /dev/null @@ -1,35 +0,0 @@ -package client - -import "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - -// setFeeForMethod sets fee for the operation executed using specified contract method. -func (x *fees) setFeeForMethod(method string, fee fixedn.Fixed8) { - if x.customFees == nil { - x.customFees = make(map[string]fixedn.Fixed8, 1) - } - - x.customFees[method] = fee -} - -// fees represents source of per-operation fees. -// Can be initialized using var declaration. -// -// Instances are not thread-safe, so they mean initially filling, and then only reading. -type fees struct { - defaultFee fixedn.Fixed8 - - // customFees represents source of customized per-operation fees. - customFees map[string]fixedn.Fixed8 -} - -// returns fee for the operation executed using specified contract method. -// Returns customized value if it is set. Otherwise, returns default value. -func (x fees) feeForMethod(method string) fixedn.Fixed8 { - if x.customFees != nil { - if fee, ok := x.customFees[method]; ok { - return fee - } - } - - return x.defaultFee -} diff --git a/pkg/morph/client/fee_test.go b/pkg/morph/client/fee_test.go deleted file mode 100644 index 963d64ce4..000000000 --- a/pkg/morph/client/fee_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package client - -import ( - "testing" - - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - "github.com/stretchr/testify/require" -) - -func TestFees(t *testing.T) { - var v fees - - const method = "some method" - - var ( - fee fixedn.Fixed8 - def = fixedn.Fixed8(13) - ) - - v.defaultFee = def - - fee = v.feeForMethod(method) - require.True(t, fee.Equal(def)) - - const customFee = fixedn.Fixed8(10) - - v.setFeeForMethod(method, customFee) - - fee = v.feeForMethod(method) - - require.Equal(t, customFee, fee) -} diff --git a/pkg/morph/client/multi.go b/pkg/morph/client/multi.go index fab90b446..e006ca69a 100644 --- a/pkg/morph/client/multi.go +++ b/pkg/morph/client/multi.go @@ -6,11 +6,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "github.com/nspcc-dev/neo-go/pkg/core/block" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/neorpc" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "go.uber.org/zap" ) @@ -34,7 +29,8 @@ func (e *endpoints) init(ee []Endpoint) { e.list = ee } -func (c *Client) switchRPC(ctx context.Context) bool { +// SwitchRPC performs reconnection and returns true if it was successful. +func (c *Client) SwitchRPC(ctx context.Context) bool { c.switchLock.Lock() defer c.switchLock.Unlock() @@ -58,20 +54,8 @@ func (c *Client) switchRPC(ctx context.Context) bool { c.logger.Info(logs.ClientConnectionToTheNewRPCNodeHasBeenEstablished, zap.String("endpoint", newEndpoint)) - subs, ok := c.restoreSubscriptions(ctx, cli, newEndpoint, false) - if !ok { - // new WS client does not allow - // restoring subscription, client - // could not work correctly => - // closing connection to RPC node - // to switch to another one - cli.Close() - continue - } - c.client = cli c.setActor(act) - c.subsInfo = subs if c.cfg.switchInterval != 0 && !c.switchIsActive.Load() && c.endpoints.list[c.endpoints.curr].Priority != c.endpoints.list[0].Priority { @@ -82,97 +66,21 @@ func (c *Client) switchRPC(ctx context.Context) bool { return true } + c.inactive = true + + if c.cfg.inactiveModeCb != nil { + c.cfg.inactiveModeCb() + } return false } -func (c *Client) notificationLoop(ctx context.Context) { - var e any - var ok bool - - for { - c.switchLock.RLock() - bChan := c.blockRcv - nChan := c.notificationRcv - nrChan := c.notaryReqRcv - c.switchLock.RUnlock() - - select { - case <-ctx.Done(): - _ = c.UnsubscribeAll() - c.close() - - return - case <-c.closeChan: - _ = c.UnsubscribeAll() - c.close() - - return - case e, ok = <-bChan: - case e, ok = <-nChan: - case e, ok = <-nrChan: - } - - if ok { - c.routeEvent(ctx, e) - continue - } - - if !c.reconnect(ctx) { - return - } - } -} - -func (c *Client) routeEvent(ctx context.Context, e any) { - typedNotification := rpcclient.Notification{Value: e} - - switch e.(type) { - case *block.Block: - typedNotification.Type = neorpc.BlockEventID - case *state.ContainedNotificationEvent: - typedNotification.Type = neorpc.NotificationEventID - case *result.NotaryRequestEvent: - typedNotification.Type = neorpc.NotaryRequestEventID - } - +func (c *Client) closeWaiter(ctx context.Context) { select { - case c.notifications <- typedNotification: case <-ctx.Done(): - _ = c.UnsubscribeAll() - c.close() case <-c.closeChan: - _ = c.UnsubscribeAll() - c.close() } -} - -func (c *Client) reconnect(ctx context.Context) bool { - if closeErr := c.client.GetError(); closeErr != nil { - c.logger.Warn(logs.ClientSwitchingToTheNextRPCNode, - zap.String("reason", closeErr.Error()), - ) - } else { - // neo-go client was closed by calling `Close` - // method, that happens only when a client has - // switched to the more prioritized RPC - return true - } - - if !c.switchRPC(ctx) { - c.logger.Error(logs.ClientCouldNotEstablishConnectionToAnyRPCNode) - - // could not connect to all endpoints => - // switch client to inactive mode - c.inactiveMode() - - return false - } - - // TODO(@carpawell): call here some callback retrieved in constructor - // of the client to allow checking chain state since during switch - // process some notification could be lost - - return true + _ = c.UnsubscribeAll() + c.close() } func (c *Client) switchToMostPrioritized(ctx context.Context) { @@ -218,36 +126,28 @@ mainLoop: continue } - if subs, ok := c.restoreSubscriptions(ctx, cli, tryE, true); ok { - c.switchLock.Lock() - - // higher priority node could have been - // connected in the other goroutine - if e.Priority >= c.endpoints.list[c.endpoints.curr].Priority { - cli.Close() - c.switchLock.Unlock() - return - } - - c.client.Close() - c.cache.invalidate() - c.client = cli - c.setActor(act) - c.subsInfo = subs - c.endpoints.curr = i + c.switchLock.Lock() + // higher priority node could have been + // connected in the other goroutine + if e.Priority >= c.endpoints.list[c.endpoints.curr].Priority { + cli.Close() c.switchLock.Unlock() - - c.logger.Info(logs.ClientSwitchedToTheHigherPriorityRPC, - zap.String("endpoint", tryE)) - return } - c.logger.Warn(logs.ClientCouldNotRestoreSideChainSubscriptionsUsingNode, - zap.String("endpoint", tryE), - zap.Error(err), - ) + c.client.Close() + c.cache.invalidate() + c.client = cli + c.setActor(act) + c.endpoints.curr = i + + c.switchLock.Unlock() + + c.logger.Info(logs.ClientSwitchedToTheHigherPriorityRPC, + zap.String("endpoint", tryE)) + + return } } } @@ -255,6 +155,7 @@ mainLoop: // close closes notification channel and wrapped WS client. func (c *Client) close() { - close(c.notifications) + c.switchLock.RLock() + defer c.switchLock.RUnlock() c.client.Close() } diff --git a/pkg/morph/client/netmap/add_peer.go b/pkg/morph/client/netmap/peer.go similarity index 70% rename from pkg/morph/client/netmap/add_peer.go rename to pkg/morph/client/netmap/peer.go index dc6c25540..7ceaa0250 100644 --- a/pkg/morph/client/netmap/add_peer.go +++ b/pkg/morph/client/netmap/peer.go @@ -41,3 +41,19 @@ func (c *Client) AddPeer(p AddPeerPrm) error { } return nil } + +// ForceRemovePeer marks the given peer as offline via a notary control transaction. +func (c *Client) ForceRemovePeer(nodeInfo netmap.NodeInfo) error { + if !c.client.WithNotary() { + return fmt.Errorf("peer can be forcefully removed only in notary environment") + } + + prm := UpdatePeerPrm{} + prm.SetKey(nodeInfo.PublicKey()) + prm.SetControlTX(true) + + if err := c.UpdatePeerState(prm); err != nil { + return fmt.Errorf("updating peer state: %v", err) + } + return nil +} diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index 2f7079dfe..53dbe180e 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" "strconv" + "time" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -81,6 +82,13 @@ func (c *Client) NNSHash() (util.Uint160, error) { return util.Uint160{}, ErrConnectionLost } + success := false + startedAt := time.Now() + + defer func() { + c.cache.metrics.AddMethodDuration("NNSContractHash", success, time.Since(startedAt)) + }() + nnsHash := c.cache.nns() if nnsHash == nil { @@ -92,6 +100,7 @@ func (c *Client) NNSHash() (util.Uint160, error) { c.cache.setNNSHash(cs.Hash) nnsHash = &cs.Hash } + success = true return *nnsHash, nil } @@ -221,7 +230,14 @@ func (c *Client) SetGroupSignerScope() error { // contractGroupKey returns public key designating FrostFS contract group. func (c *Client) contractGroupKey() (*keys.PublicKey, error) { + success := false + startedAt := time.Now() + defer func() { + c.cache.metrics.AddMethodDuration("GroupKey", success, time.Since(startedAt)) + }() + if gKey := c.cache.groupKey(); gKey != nil { + success = true return gKey, nil } @@ -251,5 +267,7 @@ func (c *Client) contractGroupKey() (*keys.PublicKey, error) { } c.cache.setGroupKey(pub) + + success = true return pub, nil } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 3e21911e1..17644361a 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -1,12 +1,14 @@ package client import ( + "crypto/elliptic" "encoding/binary" "errors" "fmt" "math" "math/big" "strings" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" @@ -18,19 +20,20 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/neorpc" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/notary" sc "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/opcode" + "github.com/nspcc-dev/neo-go/pkg/vm" + "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" "go.uber.org/zap" ) type ( notaryInfo struct { - txValidTime uint32 // minimum amount of blocks when mainTx will be valid - roundTime uint32 // extra amount of blocks to synchronize sidechain height diff of inner ring nodes - fallbackTime uint32 // mainTx's ValidUntilBlock - fallbackTime + 1 is when fallbackTx is sent + txValidTime uint32 // minimum amount of blocks when mainTx will be valid + roundTime uint32 // extra amount of blocks to synchronize sidechain height diff of inner ring nodes alphabetSource AlphabetKeys // source of alphabet node keys to prepare witness @@ -41,7 +44,7 @@ type ( notaryCfg struct { proxy util.Uint160 - txValidTime, roundTime, fallbackTime uint32 + txValidTime, roundTime uint32 alphabetSource AlphabetKeys } @@ -51,9 +54,8 @@ type ( ) const ( - defaultNotaryValidTime = 50 - defaultNotaryRoundTime = 100 - defaultNotaryFallbackTime = 40 + defaultNotaryValidTime = 50 + defaultNotaryRoundTime = 100 notaryBalanceOfMethod = "balanceOf" notaryExpirationOfMethod = "expirationOf" @@ -69,7 +71,6 @@ func defaultNotaryConfig(c *Client) *notaryCfg { return ¬aryCfg{ txValidTime: defaultNotaryValidTime, roundTime: defaultNotaryRoundTime, - fallbackTime: defaultNotaryFallbackTime, alphabetSource: c.Committee, } } @@ -104,7 +105,6 @@ func (c *Client) EnableNotarySupport(opts ...NotaryOption) error { proxy: cfg.proxy, txValidTime: cfg.txValidTime, roundTime: cfg.roundTime, - fallbackTime: cfg.fallbackTime, alphabetSource: cfg.alphabetSource, notary: notary.Hash, } @@ -408,32 +408,32 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { return fmt.Errorf("could not fetch current alphabet keys: %w", err) } - multiaddrAccount, err := c.notaryMultisigAccount(alphabetList, false, true) + cosigners, err := c.notaryCosignersFromTx(mainTx, alphabetList) if err != nil { return err } - // mainTX is expected to be pre-validated: second witness must exist and be empty - mainTx.Scripts[1].VerificationScript = multiaddrAccount.GetVerificationScript() - mainTx.Scripts[1].InvocationScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - multiaddrAccount.SignHashable(c.rpcActor.GetNetwork(), mainTx)..., - ) + nAct, err := notary.NewActor(c.client, cosigners, c.acc) + if err != nil { + return err + } + + // Sign exactly the same transaction we've got from the received Notary request. + err = nAct.Sign(mainTx) + if err != nil { + return fmt.Errorf("faield to sign notary request: %w", err) + } + + mainH, fbH, untilActual, err := nAct.Notarize(mainTx, nil) - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - resp, err := c.client.SignAndPushP2PNotaryRequest(mainTx, - []byte{byte(opcode.RET)}, - -1, - 0, - c.notary.fallbackTime, - c.acc) if err != nil && !alreadyOnChainError(err) { return err } c.logger.Debug(logs.ClientNotaryRequestWithPreparedMainTXInvoked, - zap.Uint32("fallback_valid_for", c.notary.fallbackTime), - zap.Stringer("tx_hash", resp.Hash().Reverse())) + zap.String("tx_hash", mainH.StringLE()), + zap.Uint32("valid_until_block", untilActual), + zap.String("fallback_hash", fbH.StringLE())) return nil } @@ -444,75 +444,159 @@ func (c *Client) notaryInvokeAsCommittee(method string, nonce, vub uint32, args } func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("notaryInvoke", contract.String(), method, success, time.Since(start)) + }() + alphabetList, err := c.notary.alphabetSource() if err != nil { return err } - cosigners, err := c.notaryCosigners(invokedByAlpha, alphabetList, committee) - if err != nil { - return err - } - - params, err := invocationParams(args...) - if err != nil { - return err - } - - test, err := c.makeTestInvocation(contract, method, params, cosigners) - if err != nil { - return err - } - - multiaddrAccount, err := c.notaryMultisigAccount(alphabetList, committee, invokedByAlpha) - if err != nil { - return err - } - until, err := c.getUntilValue(vub) if err != nil { return err } - mainTx, err := c.buildMainTx(invokedByAlpha, nonce, alphabetList, test, cosigners, multiaddrAccount, until) + cosigners, err := c.notaryCosigners(invokedByAlpha, alphabetList, committee) if err != nil { return err } - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - resp, err := c.client.SignAndPushP2PNotaryRequest(mainTx, - []byte{byte(opcode.RET)}, - -1, - 0, - c.notary.fallbackTime, - c.acc) + nAct, err := notary.NewActor(c.client, cosigners, c.acc) + if err != nil { + return err + } + + mainH, fbH, untilActual, err := nAct.Notarize(nAct.MakeTunedCall(contract, method, nil, func(r *result.Invoke, t *transaction.Transaction) error { + if r.State != vmstate.Halt.String() { + return wrapFrostFSError(¬HaltStateError{state: r.State, exception: r.FaultException}) + } + + t.ValidUntilBlock = until + t.Nonce = nonce + + return nil + }, args...)) + if err != nil && !alreadyOnChainError(err) { return err } c.logger.Debug(logs.ClientNotaryRequestInvoked, zap.String("method", method), - zap.Uint32("valid_until_block", until), - zap.Uint32("fallback_valid_for", c.notary.fallbackTime), - zap.Stringer("tx_hash", resp.Hash().Reverse())) + zap.Uint32("valid_until_block", untilActual), + zap.String("tx_hash", mainH.StringLE()), + zap.String("fallback_hash", fbH.StringLE())) + success = true return nil } -func (c *Client) makeTestInvocation(contract util.Uint160, method string, params []sc.Parameter, cosigners []transaction.Signer) (*result.Invoke, error) { - test, err := c.client.InvokeFunction(contract, method, params, cosigners) +func (c *Client) notaryCosignersFromTx(mainTx *transaction.Transaction, alphabetList keys.PublicKeys) ([]actor.SignerAccount, error) { + multiaddrAccount, err := c.notaryMultisigAccount(alphabetList, false, true) if err != nil { return nil, err } - if test.State != HaltState { - return nil, wrapFrostFSError(¬HaltStateError{state: test.State, exception: test.FaultException}) + // Here we need to add a committee signature (second witness) to the pre-validated + // main transaction without creating a new one. However, Notary actor demands the + // proper set of signers for constructor, thus, fill it from the main transaction's signers list. + s := make([]actor.SignerAccount, 2, 3) + s[0] = actor.SignerAccount{ + // Proxy contract that will pay for the execution. + Signer: mainTx.Signers[0], + Account: notary.FakeContractAccount(mainTx.Signers[0].Account), + } + s[1] = actor.SignerAccount{ + // Inner ring multisignature. + Signer: mainTx.Signers[1], + Account: multiaddrAccount, + } + if len(mainTx.Signers) > 3 { + // Invoker signature (simple signature account of storage node is expected). + var acc *wallet.Account + script := mainTx.Scripts[2].VerificationScript + if len(script) == 0 { + acc = notary.FakeContractAccount(mainTx.Signers[2].Account) + } else { + pubBytes, ok := vm.ParseSignatureContract(script) + if ok { + pub, err := keys.NewPublicKeyFromBytes(pubBytes, elliptic.P256()) + if err != nil { + return nil, fmt.Errorf("failed to parse verification script of signer #2: invalid public key: %w", err) + } + acc = notary.FakeSimpleAccount(pub) + } else { + m, pubsBytes, ok := vm.ParseMultiSigContract(script) + if !ok { + return nil, errors.New("failed to parse verification script of signer #2: unknown witness type") + } + pubs := make(keys.PublicKeys, len(pubsBytes)) + for i := range pubs { + pubs[i], err = keys.NewPublicKeyFromBytes(pubsBytes[i], elliptic.P256()) + if err != nil { + return nil, fmt.Errorf("failed to parse verification script of signer #2: invalid public key #%d: %w", i, err) + } + } + acc, err = notary.FakeMultisigAccount(m, pubs) + if err != nil { + return nil, fmt.Errorf("failed to create fake account for signer #2: %w", err) + } + } + } + s = append(s, actor.SignerAccount{ + Signer: mainTx.Signers[2], + Account: acc, + }) } - if len(test.Script) == 0 { - return nil, wrapFrostFSError(errEmptyInvocationScript) + return s, nil +} + +func (c *Client) notaryCosigners(invokedByAlpha bool, ir []*keys.PublicKey, committee bool) ([]actor.SignerAccount, error) { + multiaddrAccount, err := c.notaryMultisigAccount(ir, committee, invokedByAlpha) + if err != nil { + return nil, err } - return test, nil + s := make([]actor.SignerAccount, 2, 3) + // Proxy contract that will pay for the execution. + s[0] = actor.SignerAccount{ + Signer: transaction.Signer{ + Account: c.notary.proxy, + Scopes: transaction.None, + }, + Account: notary.FakeContractAccount(c.notary.proxy), + } + // Inner ring multisignature. + s[1] = actor.SignerAccount{ + Signer: transaction.Signer{ + Account: multiaddrAccount.ScriptHash(), + Scopes: c.cfg.signer.Scopes, + AllowedContracts: c.cfg.signer.AllowedContracts, + AllowedGroups: c.cfg.signer.AllowedGroups, + }, + Account: multiaddrAccount, + } + + if !invokedByAlpha { + // Invoker signature. + s = append(s, actor.SignerAccount{ + Signer: transaction.Signer{ + Account: hash.Hash160(c.acc.GetVerificationScript()), + Scopes: c.cfg.signer.Scopes, + AllowedContracts: c.cfg.signer.AllowedContracts, + AllowedGroups: c.cfg.signer.AllowedGroups, + }, + Account: c.acc, + }) + } + + // The last one is Notary contract that will be added to the signers list + // by Notary actor automatically. + return s, nil } func (c *Client) getUntilValue(vub *uint32) (uint32, error) { @@ -522,195 +606,6 @@ func (c *Client) getUntilValue(vub *uint32) (uint32, error) { return c.notaryTxValidationLimit() } -func (c *Client) buildMainTx(invokedByAlpha bool, nonce uint32, alphabetList keys.PublicKeys, test *result.Invoke, - cosigners []transaction.Signer, multiaddrAccount *wallet.Account, until uint32) (*transaction.Transaction, error) { - // after test invocation we build main multisig transaction - - u8n := uint8(len(alphabetList)) - - if !invokedByAlpha { - u8n++ - } - - // prepare main tx - mainTx := &transaction.Transaction{ - Nonce: nonce, - SystemFee: test.GasConsumed, - ValidUntilBlock: until, - Script: test.Script, - Attributes: []transaction.Attribute{ - { - Type: transaction.NotaryAssistedT, - Value: &transaction.NotaryAssisted{NKeys: u8n}, - }, - }, - Signers: cosigners, - } - - // calculate notary fee - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - notaryFee, err := c.client.CalculateNotaryFee(u8n) - if err != nil { - return nil, err - } - - // add network fee for cosigners - //nolint:staticcheck // waits for neo-go v0.99.3 with notary actors - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - err = c.client.AddNetworkFee( - mainTx, - notaryFee, - c.notaryAccounts(invokedByAlpha, multiaddrAccount)..., - ) - if err != nil { - return nil, err - } - - // define witnesses - mainTx.Scripts = c.notaryWitnesses(invokedByAlpha, multiaddrAccount, mainTx) - - return mainTx, nil -} - -func (c *Client) notaryCosigners(invokedByAlpha bool, ir []*keys.PublicKey, committee bool) ([]transaction.Signer, error) { - s := make([]transaction.Signer, 0, 4) - - // first we have proxy contract signature, as it will pay for the execution - s = append(s, transaction.Signer{ - Account: c.notary.proxy, - Scopes: transaction.None, - }) - - // then we have inner ring multiaddress signature - m := sigCount(ir, committee) - - multisigScript, err := sc.CreateMultiSigRedeemScript(m, ir) - if err != nil { - // wrap error as FrostFS-specific since the call is not related to any client - return nil, wrapFrostFSError(fmt.Errorf("can't create ir multisig redeem script: %w", err)) - } - - s = append(s, transaction.Signer{ - Account: hash.Hash160(multisigScript), - Scopes: c.cfg.signer.Scopes, - AllowedContracts: c.cfg.signer.AllowedContracts, - AllowedGroups: c.cfg.signer.AllowedGroups, - }) - - if !invokedByAlpha { - // then we have invoker signature - s = append(s, transaction.Signer{ - Account: hash.Hash160(c.acc.GetVerificationScript()), - Scopes: c.cfg.signer.Scopes, - AllowedContracts: c.cfg.signer.AllowedContracts, - AllowedGroups: c.cfg.signer.AllowedGroups, - }) - } - - // last one is a placeholder for notary contract signature - s = append(s, transaction.Signer{ - Account: c.notary.notary, - Scopes: transaction.None, - }) - - return s, nil -} - -func (c *Client) notaryAccounts(invokedByAlpha bool, multiaddr *wallet.Account) []*wallet.Account { - if multiaddr == nil { - return nil - } - - a := make([]*wallet.Account, 0, 4) - - // first we have proxy account, as it will pay for the execution - a = append(a, notary.FakeContractAccount(c.notary.proxy)) - - // then we have inner ring multiaddress account - a = append(a, multiaddr) - - if !invokedByAlpha { - // then we have invoker account - a = append(a, c.acc) - } - - // last one is a placeholder for notary contract account - a = append(a, &wallet.Account{ - Contract: &wallet.Contract{}, - }) - - return a -} - -func (c *Client) notaryWitnesses(invokedByAlpha bool, multiaddr *wallet.Account, tx *transaction.Transaction) []transaction.Witness { - if multiaddr == nil || tx == nil { - return nil - } - - w := make([]transaction.Witness, 0, 4) - - // first we have empty proxy witness, because notary will execute `Verify` - // method on the proxy contract to check witness - w = append(w, transaction.Witness{ - InvocationScript: []byte{}, - VerificationScript: []byte{}, - }) - - // then we have inner ring multiaddress witness - - // invocation script should be of the form: - // { PUSHDATA1, 64, signatureBytes... } - // to pass Notary module verification - var invokeScript []byte - - magicNumber := c.rpcActor.GetNetwork() - - if invokedByAlpha { - invokeScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - multiaddr.SignHashable(magicNumber, tx)..., - ) - } else { - // we can't provide alphabet node signature - // because Storage Node doesn't own alphabet's - // private key. Thus, add dummy witness with - // empty bytes instead of signature - invokeScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - make([]byte, 64)..., - ) - } - - w = append(w, transaction.Witness{ - InvocationScript: invokeScript, - VerificationScript: multiaddr.GetVerificationScript(), - }) - - if !invokedByAlpha { - // then we have invoker witness - invokeScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - c.acc.SignHashable(magicNumber, tx)..., - ) - - w = append(w, transaction.Witness{ - InvocationScript: invokeScript, - VerificationScript: c.acc.GetVerificationScript(), - }) - } - - // last one is a placeholder for notary contract witness - w = append(w, transaction.Witness{ - InvocationScript: append( - []byte{byte(opcode.PUSHDATA1), 64}, - make([]byte, 64)..., - ), - VerificationScript: []byte{}, - }) - - return w -} - func (c *Client) notaryMultisigAccount(ir []*keys.PublicKey, committee, invokedByAlpha bool) (*wallet.Account, error) { m := sigCount(ir, committee) @@ -767,21 +662,6 @@ func (c *Client) depositExpirationOf() (int64, error) { return currentTillBig.Int64(), nil } -func invocationParams(args ...any) ([]sc.Parameter, error) { - params := make([]sc.Parameter, 0, len(args)) - - for i := range args { - param, err := toStackParameter(args[i]) - if err != nil { - return nil, err - } - - params = append(params, param) - } - - return params, nil -} - // sigCount returns the number of required signature. // For FrostFS Alphabet M is a 2/3+1 of it (like in dBFT). // If committee is true, returns M as N/2+1. @@ -809,15 +689,6 @@ func WithRoundTime(t uint32) NotaryOption { } } -// WithFallbackTime returns a notary support option for client -// that specifies amount of blocks before fallbackTx will be sent. -// Should be less than TxValidTime. -func WithFallbackTime(t uint32) NotaryOption { - return func(c *notaryCfg) { - c.fallbackTime = t - } -} - // WithAlphabetSource returns a notary support option for client // that specifies function to return list of alphabet node keys. // By default notary subsystem uses committee as a source. This is @@ -926,7 +797,14 @@ func (c *Client) calculateNonceAndVUB(hash util.Uint256, roundBlockHeight bool) } func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { + success := false + startedAt := time.Now() + defer func() { + c.cache.metrics.AddMethodDuration("TxHeight", success, time.Since(startedAt)) + }() + if rh, ok := c.cache.txHeights.Get(h); ok { + success = true return rh, nil } height, err := c.client.GetTransactionHeight(h) @@ -934,5 +812,6 @@ func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { return 0, err } c.cache.txHeights.Add(h, height) + success = true return height, nil } diff --git a/pkg/morph/client/notifications.go b/pkg/morph/client/notifications.go index 69eafc659..dbca00d7c 100644 --- a/pkg/morph/client/notifications.go +++ b/pkg/morph/client/notifications.go @@ -1,16 +1,11 @@ package client import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/neorpc" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/util" - "go.uber.org/zap" ) // Close closes connection to the remote side making @@ -24,71 +19,46 @@ func (c *Client) Close() { close(c.closeChan) } -// SubscribeForExecutionNotifications adds subscription for notifications -// generated during contract transaction execution to this instance of client. +// ReceiveExecutionNotifications performs subscription for notifications +// generated during contract execution. Events are sent to the specified channel. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) SubscribeForExecutionNotifications(contract util.Uint160) error { +func (c *Client) ReceiveExecutionNotifications(contract util.Uint160, ch chan<- *state.ContainedNotificationEvent) (string, error) { c.switchLock.Lock() defer c.switchLock.Unlock() if c.inactive { - return ErrConnectionLost + return "", ErrConnectionLost } - _, subscribed := c.subscribedEvents[contract] - if subscribed { - // no need to subscribe one more time - return nil - } - - id, err := c.client.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, c.notificationRcv) - if err != nil { - return err - } - - c.subscribedEvents[contract] = id - - return nil + return c.client.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, ch) } -// SubscribeForNewBlocks adds subscription for new block events to this -// instance of client. +// ReceiveBlocks performs subscription for new block events. Events are sent +// to the specified channel. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) SubscribeForNewBlocks() error { +func (c *Client) ReceiveBlocks(ch chan<- *block.Block) (string, error) { c.switchLock.Lock() defer c.switchLock.Unlock() if c.inactive { - return ErrConnectionLost + return "", ErrConnectionLost } - if c.subscribedToBlocks { - // no need to subscribe one more time - return nil - } - - _, err := c.client.ReceiveBlocks(nil, c.blockRcv) - if err != nil { - return err - } - - c.subscribedToBlocks = true - - return nil + return c.client.ReceiveBlocks(nil, ch) } -// SubscribeForNotaryRequests adds subscription for notary request payloads +// ReceiveNotaryRequests performsn subscription for notary request payloads // addition or removal events to this instance of client. Passed txSigner is // used as filter: subscription is only for the notary requests that must be -// signed by txSigner. +// signed by txSigner. Events are sent to the specified channel. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) SubscribeForNotaryRequests(txSigner util.Uint160) error { +func (c *Client) ReceiveNotaryRequests(txSigner util.Uint160, ch chan<- *result.NotaryRequestEvent) (string, error) { if c.notary == nil { panic(notaryNotEnabledPanicMsg) } @@ -97,30 +67,17 @@ func (c *Client) SubscribeForNotaryRequests(txSigner util.Uint160) error { defer c.switchLock.Unlock() if c.inactive { - return ErrConnectionLost + return "", ErrConnectionLost } - _, subscribed := c.subscribedNotaryEvents[txSigner] - if subscribed { - // no need to subscribe one more time - return nil - } - - id, err := c.client.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &txSigner}, c.notaryReqRcv) - if err != nil { - return err - } - - c.subscribedNotaryEvents[txSigner] = id - - return nil + return c.client.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &txSigner}, ch) } -// UnsubscribeContract removes subscription for given contract event stream. +// Unsubscribe performs unsubscription for the given subscription ID. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) UnsubscribeContract(contract util.Uint160) error { +func (c *Client) Unsubscribe(subID string) error { c.switchLock.Lock() defer c.switchLock.Unlock() @@ -128,55 +85,7 @@ func (c *Client) UnsubscribeContract(contract util.Uint160) error { return ErrConnectionLost } - _, subscribed := c.subscribedEvents[contract] - if !subscribed { - // no need to unsubscribe contract - // without subscription - return nil - } - - err := c.client.Unsubscribe(c.subscribedEvents[contract]) - if err != nil { - return err - } - - delete(c.subscribedEvents, contract) - - return nil -} - -// UnsubscribeNotaryRequest removes subscription for given notary requests -// signer. -// -// Returns ErrConnectionLost if client has not been able to establish -// connection to any of passed RPC endpoints. -func (c *Client) UnsubscribeNotaryRequest(signer util.Uint160) error { - if c.notary == nil { - panic(notaryNotEnabledPanicMsg) - } - - c.switchLock.Lock() - defer c.switchLock.Unlock() - - if c.inactive { - return ErrConnectionLost - } - - _, subscribed := c.subscribedNotaryEvents[signer] - if !subscribed { - // no need to unsubscribe signer's - // requests without subscription - return nil - } - - err := c.client.Unsubscribe(c.subscribedNotaryEvents[signer]) - if err != nil { - return err - } - - delete(c.subscribedNotaryEvents, signer) - - return nil + return c.client.Unsubscribe(subID) } // UnsubscribeAll removes all active subscriptions of current client. @@ -191,163 +100,10 @@ func (c *Client) UnsubscribeAll() error { return ErrConnectionLost } - // no need to unsubscribe if there are - // no active subscriptions - if len(c.subscribedEvents) == 0 && len(c.subscribedNotaryEvents) == 0 && - !c.subscribedToBlocks { - return nil - } - err := c.client.UnsubscribeAll() if err != nil { return err } - c.subscribedEvents = make(map[util.Uint160]string) - c.subscribedNotaryEvents = make(map[util.Uint160]string) - c.subscribedToBlocks = false - return nil } - -// subsInfo includes channels for ws notifications; -// cached subscription information. -type subsInfo struct { - blockRcv chan *block.Block - notificationRcv chan *state.ContainedNotificationEvent - notaryReqRcv chan *result.NotaryRequestEvent - - subscribedToBlocks bool - subscribedEvents map[util.Uint160]string - subscribedNotaryEvents map[util.Uint160]string -} - -// restoreSubscriptions restores subscriptions according to cached -// information about them. -// -// If it is NOT a background operation switchLock MUST be held. -// Returns a pair: the second is a restoration status and the first -// one contains subscription information applied to the passed cli -// and receivers for the updated subscriptions. -// Does not change Client instance. -func (c *Client) restoreSubscriptions(ctx context.Context, cli *rpcclient.WSClient, endpoint string, background bool) (si subsInfo, ok bool) { - var ( - err error - id string - ) - - stopCh := make(chan struct{}) - defer close(stopCh) - - blockRcv := make(chan *block.Block) - notificationRcv := make(chan *state.ContainedNotificationEvent) - notaryReqRcv := make(chan *result.NotaryRequestEvent) - - c.startListen(ctx, stopCh, blockRcv, notificationRcv, notaryReqRcv, background) - - if background { - c.switchLock.RLock() - defer c.switchLock.RUnlock() - } - - si.subscribedToBlocks = c.subscribedToBlocks - si.subscribedEvents = copySubsMap(c.subscribedEvents) - si.subscribedNotaryEvents = copySubsMap(c.subscribedNotaryEvents) - si.blockRcv = blockRcv - si.notificationRcv = notificationRcv - si.notaryReqRcv = notaryReqRcv - - // new block events restoration - if si.subscribedToBlocks { - _, err = cli.ReceiveBlocks(nil, blockRcv) - if err != nil { - c.logger.Error(logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, - zap.String("endpoint", endpoint), - zap.Error(err), - ) - - return - } - } - - // notification events restoration - for contract := range si.subscribedEvents { - contract := contract // See https://github.com/nspcc-dev/neo-go/issues/2890 - id, err = cli.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, notificationRcv) - if err != nil { - c.logger.Error(logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, - zap.String("endpoint", endpoint), - zap.Error(err), - ) - - return - } - - si.subscribedEvents[contract] = id - } - - // notary notification events restoration - if c.notary != nil { - for signer := range si.subscribedNotaryEvents { - signer := signer // See https://github.com/nspcc-dev/neo-go/issues/2890 - id, err = cli.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &signer}, notaryReqRcv) - if err != nil { - c.logger.Error(logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, - zap.String("endpoint", endpoint), - zap.Error(err), - ) - - return - } - - si.subscribedNotaryEvents[signer] = id - } - } - - return si, true -} - -func (c *Client) startListen(ctx context.Context, stopCh <-chan struct{}, blockRcv <-chan *block.Block, - notificationRcv <-chan *state.ContainedNotificationEvent, notaryReqRcv <-chan *result.NotaryRequestEvent, background bool) { - // neo-go WS client says to _always_ read notifications - // from its channel. Subscribing to any notification - // while not reading them in another goroutine may - // lead to a dead-lock, thus that async side notification - // listening while restoring subscriptions - - go func() { - var e any - var ok bool - - for { - select { - case <-stopCh: - return - case e, ok = <-blockRcv: - case e, ok = <-notificationRcv: - case e, ok = <-notaryReqRcv: - } - - if !ok { - return - } - - if background { - // background client (test) switch, no need to send - // any notification, just preventing dead-lock - continue - } - - c.routeEvent(ctx, e) - } - }() -} - -func copySubsMap(m map[util.Uint160]string) map[util.Uint160]string { - newM := make(map[util.Uint160]string, len(m)) - for k, v := range m { - newM[k] = v - } - - return newM -} diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index 910f78537..7aa17a70f 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -27,7 +27,7 @@ type staticOpts struct { tryNotary bool alpha bool // use client's key to sign notary request's main TX - fees fees + fee fixedn.Fixed8 } // WithNotary returns notary status of the client. @@ -63,7 +63,7 @@ func NewStatic(client *Client, scriptHash util.Uint160, fee fixedn.Fixed8, opts scScriptHash: scriptHash, } - c.fees.defaultFee = fee + c.fee = fee for i := range opts { opts[i](&c.staticOpts) @@ -125,8 +125,6 @@ func (i *InvokePrmOptional) SetControlTX(b bool) { // If fee for the operation executed using specified method is customized, then StaticClient uses it. // Otherwise, default fee is used. func (s StaticClient) Invoke(prm InvokePrm) error { - fee := s.fees.feeForMethod(prm.method) - if s.tryNotary { if s.alpha { var ( @@ -149,15 +147,15 @@ func (s StaticClient) Invoke(prm InvokePrm) error { vubP = &vub } - return s.client.NotaryInvoke(s.scScriptHash, fee, nonce, vubP, prm.method, prm.args...) + return s.client.NotaryInvoke(s.scScriptHash, s.fee, nonce, vubP, prm.method, prm.args...) } - return s.client.NotaryInvokeNotAlpha(s.scScriptHash, fee, prm.method, prm.args...) + return s.client.NotaryInvokeNotAlpha(s.scScriptHash, s.fee, prm.method, prm.args...) } return s.client.Invoke( s.scScriptHash, - fee, + s.fee, prm.method, prm.args..., ) @@ -211,11 +209,3 @@ func AsAlphabet() StaticClientOption { o.alpha = true } } - -// WithCustomFee returns option to specify custom fee for the operation executed using -// specified contract method. -func WithCustomFee(method string, fee fixedn.Fixed8) StaticClientOption { - return func(o *staticOpts) { - o.fees.setFeeForMethod(method, fee) - } -} diff --git a/pkg/morph/event/container/delete.go b/pkg/morph/event/container/delete.go index 7286ddcfc..a206307f8 100644 --- a/pkg/morph/event/container/delete.go +++ b/pkg/morph/event/container/delete.go @@ -15,6 +15,7 @@ type Delete struct { ContainerIDValue []byte SignatureValue []byte TokenValue []byte + PublicKeyValue []byte // For notary notifications only. // Contains raw transactions of notary request. @@ -42,46 +43,7 @@ func (d Delete) NotaryRequest() *payload.P2PNotaryRequest { return d.NotaryRequestValue } -const expectedItemNumDelete = 3 - -// ParseDelete from notification into container event structure. -// -// Expects 3 stack items. -func ParseDelete(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Delete - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumDelete { - return nil, event.WrongNumberOfParameters(expectedItemNumDelete, ln) - } - - // parse container - ev.ContainerIDValue, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get container: %w", err) - } - - // parse signature - ev.SignatureValue, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get signature: %w", err) - } - - // parse session token - ev.TokenValue, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get session token: %w", err) - } - - return ev, nil -} +const expectedItemNumDelete = 4 // DeleteSuccess structures notification event of successful container removal // thrown by Container contract. diff --git a/pkg/morph/event/container/delete_notary.go b/pkg/morph/event/container/delete_notary.go index 23f13acbb..9711636e7 100644 --- a/pkg/morph/event/container/delete_notary.go +++ b/pkg/morph/event/container/delete_notary.go @@ -17,6 +17,10 @@ func (d *Delete) setSignature(v []byte) { } } +func (d *Delete) setPublicKey(v []byte) { + d.PublicKeyValue = v +} + func (d *Delete) setToken(v []byte) { if v != nil { d.TokenValue = v @@ -26,6 +30,7 @@ func (d *Delete) setToken(v []byte) { var deleteFieldSetters = []func(*Delete, []byte){ // order on stack is reversed (*Delete).setToken, + (*Delete).setPublicKey, (*Delete).setSignature, (*Delete).setContainerID, } diff --git a/pkg/morph/event/container/delete_test.go b/pkg/morph/event/container/delete_test.go index 782f4aade..627c5fcf5 100644 --- a/pkg/morph/event/container/delete_test.go +++ b/pkg/morph/event/container/delete_test.go @@ -10,66 +10,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestParseDelete(t *testing.T) { - var ( - containerID = []byte("containreID") - signature = []byte("signature") - token = []byte("token") - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - } - - _, err := ParseDelete(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(3, len(prms)).Error()) - }) - - t.Run("wrong container parameter", func(t *testing.T) { - _, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong signature parameter", func(t *testing.T) { - _, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerID), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong session token parameter", func(t *testing.T) { - _, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerID), - stackitem.NewByteArray(signature), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerID), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(token), - })) - - require.NoError(t, err) - - require.Equal(t, Delete{ - ContainerIDValue: containerID, - SignatureValue: signature, - TokenValue: token, - }, ev) - }) -} - func TestParseDeleteSuccess(t *testing.T) { t.Run("wrong number of parameters", func(t *testing.T) { prms := []stackitem.Item{ diff --git a/pkg/morph/event/container/eacl.go b/pkg/morph/event/container/eacl.go index 41058ea43..4168d8842 100644 --- a/pkg/morph/event/container/eacl.go +++ b/pkg/morph/event/container/eacl.go @@ -1,11 +1,6 @@ package container import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/network/payload" ) @@ -54,48 +49,3 @@ func (x SetEACL) NotaryRequest() *payload.P2PNotaryRequest { } const expectedItemNumEACL = 4 - -// ParseSetEACL parses SetEACL notification event from list of stack items. -// -// Expects 4 stack items. -func ParseSetEACL(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev SetEACL - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumEACL { - return nil, event.WrongNumberOfParameters(expectedItemNumEACL, ln) - } - - // parse table - ev.TableValue, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not parse binary table: %w", err) - } - - // parse signature - ev.SignatureValue, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not parse table signature: %w", err) - } - - // parse public key - ev.PublicKeyValue, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not parse binary public key: %w", err) - } - - // parse session token - ev.TokenValue, err = client.BytesFromStackItem(params[3]) - if err != nil { - return nil, fmt.Errorf("could not get session token: %w", err) - } - - return ev, nil -} diff --git a/pkg/morph/event/container/eacl_test.go b/pkg/morph/event/container/eacl_test.go index 2f0598597..159f6cd9f 100644 --- a/pkg/morph/event/container/eacl_test.go +++ b/pkg/morph/event/container/eacl_test.go @@ -1,90 +1,10 @@ package container import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" ) -func TestParseEACL(t *testing.T) { - var ( - binaryTable = []byte("table") - signature = []byte("signature") - publicKey = []byte("pubkey") - token = []byte("token") - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - items := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParseSetEACL(createNotifyEventFromItems(items)) - require.EqualError(t, err, event.WrongNumberOfParameters(4, len(items)).Error()) - }) - - t.Run("wrong container parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong signature parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong key parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewByteArray(signature), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong session token parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewByteArray(token), - })) - require.NoError(t, err) - - e := ev.(SetEACL) - - require.Equal(t, binaryTable, e.Table()) - require.Equal(t, signature, e.Signature()) - require.Equal(t, publicKey, e.PublicKey()) - require.Equal(t, token, e.SessionToken()) - }) -} - func createNotifyEventFromItems(items []stackitem.Item) *state.ContainedNotificationEvent { return &state.ContainedNotificationEvent{ NotificationEvent: state.NotificationEvent{ diff --git a/pkg/morph/event/container/put.go b/pkg/morph/event/container/put.go index d163c6836..335034bf3 100644 --- a/pkg/morph/event/container/put.go +++ b/pkg/morph/event/container/put.go @@ -65,49 +65,6 @@ func (x PutNamed) Zone() string { return x.zone } -// ParsePut from notification into container event structure. -func ParsePut(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Put - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumPut { - return nil, event.WrongNumberOfParameters(expectedItemNumPut, ln) - } - - // parse container - ev.rawContainer, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get container: %w", err) - } - - // parse signature - ev.signature, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get signature: %w", err) - } - - // parse public key - ev.publicKey, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get public key: %w", err) - } - - // parse session token - ev.token, err = client.BytesFromStackItem(params[3]) - if err != nil { - return nil, fmt.Errorf("could not get sesison token: %w", err) - } - - return ev, nil -} - // PutSuccess structures notification event of successful container creation // thrown by Container contract. type PutSuccess struct { diff --git a/pkg/morph/event/container/put_test.go b/pkg/morph/event/container/put_test.go index 2ccea296f..3622f9943 100644 --- a/pkg/morph/event/container/put_test.go +++ b/pkg/morph/event/container/put_test.go @@ -10,80 +10,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestParsePut(t *testing.T) { - var ( - containerData = []byte("containerData") - signature = []byte("signature") - publicKey = []byte("pubkey") - token = []byte("token") - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParsePut(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(expectedItemNumPut, len(prms)).Error()) - }) - - t.Run("wrong container parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong signature parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong key parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewByteArray(signature), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong session token parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewByteArray(token), - })) - require.NoError(t, err) - - require.Equal(t, Put{ - rawContainer: containerData, - signature: signature, - publicKey: publicKey, - token: token, - }, ev) - }) -} - func TestParsePutSuccess(t *testing.T) { t.Run("wrong number of parameters", func(t *testing.T) { prms := []stackitem.Item{ diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 405165702..ca5031415 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -365,11 +365,14 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { // prepare the notary event notaryEvent, err := l.notaryEventsPreparator.Prepare(nr.NotaryRequest) if err != nil { + var expErr *ExpiredTXError switch { case errors.Is(err, ErrTXAlreadyHandled): - case errors.Is(err, ErrMainTXExpired): + case errors.As(err, &expErr): l.log.Warn(logs.EventSkipExpiredMainTXNotaryEvent, zap.String("error", err.Error()), + zap.Uint32("current_block_height", expErr.CurrentBlockHeight), + zap.Uint32("fallback_tx_not_valid_before_height", expErr.FallbackTXNotValidBeforeHeight), ) default: l.log.Warn(logs.EventCouldNotPrepareAndValidateNotaryEvent, diff --git a/pkg/morph/event/netmap/add_peer.go b/pkg/morph/event/netmap/add_peer.go index 6f839bada..80c5559fc 100644 --- a/pkg/morph/event/netmap/add_peer.go +++ b/pkg/morph/event/netmap/add_peer.go @@ -1,11 +1,6 @@ package netmap import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/network/payload" ) @@ -31,26 +26,3 @@ func (s AddPeer) NotaryRequest() *payload.P2PNotaryRequest { } const expectedItemNumAddPeer = 1 - -func ParseAddPeer(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev AddPeer - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumAddPeer { - return nil, event.WrongNumberOfParameters(expectedItemNumAddPeer, ln) - } - - ev.NodeBytes, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get raw nodeinfo: %w", err) - } - - return ev, nil -} diff --git a/pkg/morph/event/netmap/add_peer_test.go b/pkg/morph/event/netmap/add_peer_test.go index 0574c4048..4118bb8c8 100644 --- a/pkg/morph/event/netmap/add_peer_test.go +++ b/pkg/morph/event/netmap/add_peer_test.go @@ -1,47 +1,10 @@ package netmap import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" ) -func TestParseAddPeer(t *testing.T) { - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParseAddPeer(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(1, len(prms)).Error()) - }) - - t.Run("wrong first parameter type", func(t *testing.T) { - _, err := ParseAddPeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - info := []byte{1, 2, 3} - - ev, err := ParseAddPeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(info), - })) - - require.NoError(t, err) - require.Equal(t, AddPeer{ - NodeBytes: info, - }, ev) - }) -} - func createNotifyEventFromItems(items []stackitem.Item) *state.ContainedNotificationEvent { return &state.ContainedNotificationEvent{ NotificationEvent: state.NotificationEvent{ diff --git a/pkg/morph/event/netmap/update_peer.go b/pkg/morph/event/netmap/update_peer.go index f02ca408d..e29671131 100644 --- a/pkg/morph/event/netmap/update_peer.go +++ b/pkg/morph/event/netmap/update_peer.go @@ -1,13 +1,9 @@ package netmap import ( - "crypto/elliptic" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/network/payload" ) @@ -60,43 +56,3 @@ func (s *UpdatePeer) decodeState(state int64) error { } const expectedItemNumUpdatePeer = 2 - -func ParseUpdatePeer(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev UpdatePeer - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumUpdatePeer { - return nil, event.WrongNumberOfParameters(expectedItemNumUpdatePeer, ln) - } - - // parse public key - key, err := client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get public key: %w", err) - } - - ev.PubKey, err = keys.NewPublicKeyFromBytes(key, elliptic.P256()) - if err != nil { - return nil, fmt.Errorf("could not parse public key: %w", err) - } - - // parse node status - st, err := client.IntFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get node status: %w", err) - } - - err = ev.decodeState(st) - if err != nil { - return nil, err - } - - return ev, nil -} diff --git a/pkg/morph/event/netmap/update_peer_test.go b/pkg/morph/event/netmap/update_peer_test.go deleted file mode 100644 index b79dd6385..000000000 --- a/pkg/morph/event/netmap/update_peer_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package netmap - -import ( - "math/big" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func TestParseUpdatePeer(t *testing.T) { - priv, err := keys.NewPrivateKey() - require.NoError(t, err) - - publicKey := priv.PublicKey() - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - } - - _, err := ParseUpdatePeer(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(2, len(prms)).Error()) - }) - - t.Run("wrong first parameter type", func(t *testing.T) { - _, err := ParseUpdatePeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong second parameter type", func(t *testing.T) { - _, err := ParseUpdatePeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(publicKey.Bytes()), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - const state = netmap.NodeStateMaintenance - ev, err := ParseUpdatePeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewBigInteger(big.NewInt(int64(state))), - stackitem.NewByteArray(publicKey.Bytes()), - })) - require.NoError(t, err) - - require.Equal(t, UpdatePeer{ - PubKey: publicKey, - State: state, - }, ev) - }) -} diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index f7b10d906..298a6d574 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -39,11 +39,18 @@ var ( // ErrTXAlreadyHandled is returned if received TX has already been signed. ErrTXAlreadyHandled = errors.New("received main tx has already been handled") - - // ErrMainTXExpired is returned if received fallback TX is already valid. - ErrMainTXExpired = errors.New("received main tx has expired") ) +// ExpiredTXError is returned if received fallback TX is already valid. +type ExpiredTXError struct { + CurrentBlockHeight uint32 + FallbackTXNotValidBeforeHeight uint32 +} + +func (e *ExpiredTXError) Error() string { + return "received main tx has expired" +} + // BlockCounter must return block count of the network // from which notary requests are received. type BlockCounter interface { @@ -185,15 +192,15 @@ func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { } invokerWitness := ln == 4 - multiInvScript := nr.MainTransaction.Scripts[1].InvocationScript - - // alphabet node should handle only notary requests - // that have been sent unsigned (by storage nodes) => - // such main TXs should have either a dummy or an - // empty script as an invocation script + // alphabet node should handle only notary requests that do not yet have inner + // ring multisignature filled => such main TXs either have empty invocation script + // of the inner ring witness (in case if Notary Actor is used to create request) + // or have it filled with dummy bytes (if request was created manually with the old + // neo-go API) // // this check prevents notary flow recursion - if len(multiInvScript) > 0 && !bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript) { + if !(len(nr.MainTransaction.Scripts[1].InvocationScript) == 0 || + bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript)) { // compatibility with old version return ErrTXAlreadyHandled } @@ -220,12 +227,7 @@ func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { } // validate main TX expiration - err = p.validateExpiration(nr.FallbackTransaction) - if err != nil { - return err - } - - return nil + return p.validateExpiration(nr.FallbackTransaction) } func (p Preparator) validateParameterOpcodes(ops []Op) error { @@ -313,7 +315,10 @@ func (p Preparator) validateExpiration(fbTX *transaction.Transaction) error { } if currBlock >= nvb.Height { - return ErrMainTXExpired + return &ExpiredTXError{ + CurrentBlockHeight: currBlock, + FallbackTXNotValidBeforeHeight: nvb.Height, + } } return nil @@ -363,7 +368,9 @@ func (p Preparator) validateWitnesses(w []transaction.Witness, alphaKeys keys.Pu // the last one must be a placeholder for notary contract witness last := len(w) - 1 - if !bytes.Equal(w[last].InvocationScript, p.dummyInvocationScript) || len(w[last].VerificationScript) != 0 { + if !(len(w[last].InvocationScript) == 0 || // https://github.com/nspcc-dev/neo-go/pull/2981 + bytes.Equal(w[last].InvocationScript, p.dummyInvocationScript)) || // compatibility with old version + len(w[last].VerificationScript) != 0 { return errIncorrectNotaryPlaceholder } diff --git a/pkg/morph/event/notary_preparator_test.go b/pkg/morph/event/notary_preparator_test.go index d0463348d..8da9d868a 100644 --- a/pkg/morph/event/notary_preparator_test.go +++ b/pkg/morph/event/notary_preparator_test.go @@ -1,6 +1,7 @@ package event import ( + "fmt" "testing" "github.com/nspcc-dev/neo-go/pkg/vm" @@ -24,8 +25,9 @@ var ( alphaKeys keys.PublicKeys wrongAlphaKeys keys.PublicKeys - dummyInvocationScript = append([]byte{byte(opcode.PUSHDATA1), 64}, make([]byte, 64)...) - wrongDummyInvocationScript = append([]byte{byte(opcode.PUSHDATA1), 64, 1}, make([]byte, 63)...) + dummyAlphabetInvocationScript = []byte{} // expected to be empty if generated by Notary Actor, as requester can't fill it in + dummyAlphabetInvocationScriptOld = append([]byte{byte(opcode.PUSHDATA1), 64}, make([]byte, 64)...) // expected to be dummy if generated manually + wrongDummyInvocationScript = append([]byte{byte(opcode.PUSHDATA1), 64, 1}, make([]byte, 63)...) scriptHash util.Uint160 ) @@ -61,35 +63,37 @@ func TestPrepare_IncorrectScript(t *testing.T) { }, ) - t.Run("not contract call", func(t *testing.T) { - bw := io.NewBufBinWriter() + for _, dummyMultisig := range []bool{true, false} { // try both empty and dummy multisig/Notary invocation witness script + t.Run(fmt.Sprintf("not contract call, compat: %t", dummyMultisig), func(t *testing.T) { + bw := io.NewBufBinWriter() - emit.Int(bw.BinWriter, 4) - emit.String(bw.BinWriter, "test") - emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) - emit.Syscall(bw.BinWriter, interopnames.SystemContractCallNative) // any != interopnames.SystemContractCall + emit.Int(bw.BinWriter, 4) + emit.String(bw.BinWriter, "test") + emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) + emit.Syscall(bw.BinWriter, interopnames.SystemContractCallNative) // any != interopnames.SystemContractCall - nr := correctNR(bw.Bytes(), false) + nr := correctNR(bw.Bytes(), dummyMultisig, false) - _, err := preparator.Prepare(nr) + _, err := preparator.Prepare(nr) - require.EqualError(t, err, errNotContractCall.Error()) - }) + require.EqualError(t, err, errNotContractCall.Error()) + }) - t.Run("incorrect ", func(t *testing.T) { - bw := io.NewBufBinWriter() + t.Run(fmt.Sprintf("incorrect, compat: %t", dummyMultisig), func(t *testing.T) { + bw := io.NewBufBinWriter() - emit.Int(bw.BinWriter, -1) - emit.String(bw.BinWriter, "test") - emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) - emit.Syscall(bw.BinWriter, interopnames.SystemContractCall) + emit.Int(bw.BinWriter, -1) + emit.String(bw.BinWriter, "test") + emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) + emit.Syscall(bw.BinWriter, interopnames.SystemContractCall) - nr := correctNR(bw.Bytes(), false) + nr := correctNR(bw.Bytes(), dummyMultisig, false) - _, err := preparator.Prepare(nr) + _, err := preparator.Prepare(nr) - require.EqualError(t, err, errIncorrectCallFlag.Error()) - }) + require.EqualError(t, err, errIncorrectCallFlag.Error()) + }) + } } func TestPrepare_IncorrectNR(t *testing.T) { @@ -209,7 +213,23 @@ func TestPrepare_IncorrectNR(t *testing.T) { InvocationScript: make([]byte, 1), }, { - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + {}, + }, + }, + expErr: errIncorrectProxyWitnesses, + }, + { + name: "incorrect main TX proxy witness compat", + addW: false, + mTX: mTX{ + scripts: []transaction.Witness{ + { + InvocationScript: make([]byte, 1), + }, + { + InvocationScript: dummyAlphabetInvocationScriptOld, }, {}, }, @@ -224,7 +244,22 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, { VerificationScript: wrongAlphaVerificationScript, - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + {}, + }, + }, + expErr: errIncorrectAlphabet, + }, + { + name: "incorrect main TX Alphabet witness compat", + addW: false, + mTX: mTX{ + scripts: []transaction.Witness{ + {}, + { + VerificationScript: wrongAlphaVerificationScript, + InvocationScript: dummyAlphabetInvocationScriptOld, }, {}, }, @@ -239,7 +274,24 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, { VerificationScript: alphaVerificationScript, - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + { + InvocationScript: wrongDummyInvocationScript, + }, + }, + }, + expErr: errIncorrectNotaryPlaceholder, + }, + { + name: "incorrect main TX Notary witness compat", + addW: false, + mTX: mTX{ + scripts: []transaction.Witness{ + {}, + { + VerificationScript: alphaVerificationScript, + InvocationScript: dummyAlphabetInvocationScriptOld, }, { InvocationScript: wrongDummyInvocationScript, @@ -279,7 +331,7 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, }, }, - expErr: ErrMainTXExpired, + expErr: &ExpiredTXError{}, }, { name: "incorrect invoker TX Alphabet witness", @@ -289,7 +341,23 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, { VerificationScript: alphaVerificationScript, - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + {}, + {}, + }, + }, + expErr: errIncorrectInvokerWitnesses, + }, + { + name: "incorrect invoker TX Alphabet witness compat", + addW: true, + mTX: mTX{ + scripts: []transaction.Witness{ + {}, + { + VerificationScript: alphaVerificationScript, + InvocationScript: dummyAlphabetInvocationScriptOld, }, {}, {}, @@ -327,7 +395,7 @@ func TestPrepare_IncorrectNR(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - correctNR := correctNR(nil, test.addW) + correctNR := correctNR(nil, false, test.addW) incorrectNR = setIncorrectFields(*correctNR, test.mTX, test.fbTX) _, err = preparator.Prepare(&incorrectNR) @@ -372,40 +440,42 @@ func TestPrepare_CorrectNR(t *testing.T) { for _, test := range tests { for i := 0; i < 1; i++ { // run tests against 3 and 4 witness NR - additionalWitness := i == 0 - nr := correctNR(script(test.hash, test.method, test.args...), additionalWitness) + for _, dummyMultisig := range []bool{true, false} { // run tests against empty and dummy multisig/Notary witness + additionalWitness := i == 0 + nr := correctNR(script(test.hash, test.method, test.args...), dummyMultisig, additionalWitness) - event, err := preparator.Prepare(nr) + event, err := preparator.Prepare(nr) - require.NoError(t, err) - require.Equal(t, test.method, event.Type().String()) - require.Equal(t, test.hash.StringLE(), event.ScriptHash().StringLE()) - - // check args parsing - bw := io.NewBufBinWriter() - emit.Array(bw.BinWriter, test.args...) - - ctx := vm.NewContext(bw.Bytes()) - - opCode, param, err := ctx.Next() - require.NoError(t, err) - - for _, opGot := range event.Params() { - require.Equal(t, opCode, opGot.code) - require.Equal(t, param, opGot.param) - - opCode, param, err = ctx.Next() require.NoError(t, err) + require.Equal(t, test.method, event.Type().String()) + require.Equal(t, test.hash.StringLE(), event.ScriptHash().StringLE()) + + // check args parsing + bw := io.NewBufBinWriter() + emit.Array(bw.BinWriter, test.args...) + + ctx := vm.NewContext(bw.Bytes()) + + opCode, param, err := ctx.Next() + require.NoError(t, err) + + for _, opGot := range event.Params() { + require.Equal(t, opCode, opGot.code) + require.Equal(t, param, opGot.param) + + opCode, param, err = ctx.Next() + require.NoError(t, err) + } + + _, _, err = ctx.Next() // PACK opcode + require.NoError(t, err) + _, _, err = ctx.Next() // packing len opcode + require.NoError(t, err) + + opCode, _, err = ctx.Next() + require.NoError(t, err) + require.Equal(t, opcode.RET, opCode) } - - _, _, err = ctx.Next() // PACK opcode - require.NoError(t, err) - _, _, err = ctx.Next() // packing len opcode - require.NoError(t, err) - - opCode, _, err = ctx.Next() - require.NoError(t, err) - require.Equal(t, opcode.RET, opCode) } } } @@ -428,7 +498,7 @@ func script(hash util.Uint160, method string, args ...any) []byte { return bw.Bytes() } -func correctNR(script []byte, additionalWitness bool) *payload.P2PNotaryRequest { +func correctNR(script []byte, dummyMultisig, additionalWitness bool) *payload.P2PNotaryRequest { alphaVerificationScript, _ := smartcontract.CreateMultiSigRedeemScript(len(alphaKeys)*2/3+1, alphaKeys) signers := []transaction.Signer{ @@ -443,20 +513,24 @@ func correctNR(script []byte, additionalWitness bool) *payload.P2PNotaryRequest signers[2] = transaction.Signer{Account: hash.Hash160(alphaVerificationScript)} } + multisigInv := dummyAlphabetInvocationScript + if dummyMultisig { + multisigInv = dummyAlphabetInvocationScriptOld + } scripts := []transaction.Witness{ {}, { - InvocationScript: dummyInvocationScript, + InvocationScript: multisigInv, VerificationScript: alphaVerificationScript, }, { - InvocationScript: dummyInvocationScript, + InvocationScript: multisigInv, }, } if additionalWitness { // insert on element with index 2 scripts = append(scripts[:2+1], scripts[2:]...) scripts[2] = transaction.Witness{ - InvocationScript: dummyInvocationScript, + InvocationScript: multisigInv, VerificationScript: alphaVerificationScript, } } diff --git a/pkg/morph/metrics/metrics.go b/pkg/morph/metrics/metrics.go new file mode 100644 index 000000000..5d74b054d --- /dev/null +++ b/pkg/morph/metrics/metrics.go @@ -0,0 +1,21 @@ +package metrics + +import "time" + +type Register interface { + IncSwitchCount() + SetLastBlock(uint32) + IncNotificationCount(notificationType string) + ObserveInvoke(typ string, contract string, method string, success bool, d time.Duration) +} + +type NoopRegister struct{} + +func (NoopRegister) IncSwitchCount() {} +func (NoopRegister) SetLastBlock(uint32) {} +func (NoopRegister) IncNotificationCount(string) {} +func (NoopRegister) ObserveInvoke(string, string, string, bool, time.Duration) {} + +type NoopMorphCacheMetrics struct{} + +func (m *NoopMorphCacheMetrics) AddMethodDuration(string, bool, time.Duration) {} diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index a2e1c32eb..4076111f0 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -11,8 +11,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/neorpc" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" ) @@ -27,7 +27,6 @@ type ( // Subscriber is an interface of the NotificationEvent listener. Subscriber interface { SubscribeForNotification(...util.Uint160) error - UnsubscribeForNotification() BlockNotifications() error SubscribeForNotaryRequests(mainTXSigner util.Uint160) error @@ -36,16 +35,27 @@ type ( Close() } + subChannels struct { + NotifyChan chan *state.ContainedNotificationEvent + BlockChan chan *block.Block + NotaryChan chan *result.NotaryRequestEvent + } + subscriber struct { - *sync.RWMutex + sync.RWMutex log *logger.Logger client *client.Client notifyChan chan *state.ContainedNotificationEvent - - blockChan chan *block.Block - + blockChan chan *block.Block notaryChan chan *result.NotaryRequestEvent + + current subChannels + + // cached subscription information + subscribedEvents map[util.Uint160]bool + subscribedNotaryEvents map[util.Uint160]bool + subscribedToNewBlocks bool } // Params is a group of Subscriber constructor parameters. @@ -76,116 +86,66 @@ func (s *subscriber) SubscribeForNotification(contracts ...util.Uint160) error { s.Lock() defer s.Unlock() - notifyIDs := make(map[util.Uint160]struct{}, len(contracts)) + notifyIDs := make([]string, 0, len(contracts)) for i := range contracts { + if s.subscribedEvents[contracts[i]] { + continue + } // subscribe to contract notifications - err := s.client.SubscribeForExecutionNotifications(contracts[i]) + id, err := s.client.ReceiveExecutionNotifications(contracts[i], s.current.NotifyChan) if err != nil { // if there is some error, undo all subscriptions and return error - for hash := range notifyIDs { - _ = s.client.UnsubscribeContract(hash) + for _, id := range notifyIDs { + _ = s.client.Unsubscribe(id) } return err } // save notification id - notifyIDs[contracts[i]] = struct{}{} + notifyIDs = append(notifyIDs, id) + } + for i := range contracts { + s.subscribedEvents[contracts[i]] = true } return nil } -func (s *subscriber) UnsubscribeForNotification() { - err := s.client.UnsubscribeAll() - if err != nil { - s.log.Error(logs.SubscriberUnsubscribeForNotification, - zap.Error(err)) - } -} - func (s *subscriber) Close() { s.client.Close() } func (s *subscriber) BlockNotifications() error { - if err := s.client.SubscribeForNewBlocks(); err != nil { + s.Lock() + defer s.Unlock() + if s.subscribedToNewBlocks { + return nil + } + if _, err := s.client.ReceiveBlocks(s.current.BlockChan); err != nil { return fmt.Errorf("could not subscribe for new block events: %w", err) } + s.subscribedToNewBlocks = true + return nil } func (s *subscriber) SubscribeForNotaryRequests(mainTXSigner util.Uint160) error { - if err := s.client.SubscribeForNotaryRequests(mainTXSigner); err != nil { + s.Lock() + defer s.Unlock() + if s.subscribedNotaryEvents[mainTXSigner] { + return nil + } + if _, err := s.client.ReceiveNotaryRequests(mainTXSigner, s.current.NotaryChan); err != nil { return fmt.Errorf("could not subscribe for notary request events: %w", err) } + s.subscribedNotaryEvents[mainTXSigner] = true return nil } -func (s *subscriber) routeNotifications(ctx context.Context) { - notificationChan := s.client.NotificationChannel() - - for { - select { - case <-ctx.Done(): - return - case notification, ok := <-notificationChan: - if !ok { - s.log.Warn(logs.SubscriberRemoteNotificationChannelHasBeenClosed) - close(s.notifyChan) - close(s.blockChan) - close(s.notaryChan) - - return - } - - switch notification.Type { - case neorpc.NotificationEventID: - notifyEvent, ok := notification.Value.(*state.ContainedNotificationEvent) - if !ok { - s.log.Error(logs.SubscriberCantCastNotifyEventValueToTheNotifyStruct, - zap.String("received type", fmt.Sprintf("%T", notification.Value)), - ) - continue - } - - s.log.Debug(logs.SubscriberNewNotificationEventFromSidechain, - zap.String("name", notifyEvent.Name), - ) - - s.notifyChan <- notifyEvent - case neorpc.BlockEventID: - b, ok := notification.Value.(*block.Block) - if !ok { - s.log.Error(logs.SubscriberCantCastBlockEventValueToBlock, - zap.String("received type", fmt.Sprintf("%T", notification.Value)), - ) - continue - } - - s.blockChan <- b - case neorpc.NotaryRequestEventID: - notaryRequest, ok := notification.Value.(*result.NotaryRequestEvent) - if !ok { - s.log.Error(logs.SubscriberCantCastNotifyEventValueToTheNotaryRequestStruct, - zap.String("received type", fmt.Sprintf("%T", notification.Value)), - ) - continue - } - - s.notaryChan <- notaryRequest - default: - s.log.Debug(logs.SubscriberUnsupportedNotificationFromTheChain, - zap.Uint8("type", uint8(notification.Type)), - ) - } - } - } -} - // New is a constructs Neo:Morph event listener and returns Subscriber interface. func New(ctx context.Context, p *Params) (Subscriber, error) { switch { @@ -203,22 +163,180 @@ func New(ctx context.Context, p *Params) (Subscriber, error) { } sub := &subscriber{ - RWMutex: new(sync.RWMutex), log: p.Log, client: p.Client, notifyChan: make(chan *state.ContainedNotificationEvent), blockChan: make(chan *block.Block), notaryChan: make(chan *result.NotaryRequestEvent), - } - // Worker listens all events from neo-go websocket and puts them - // into corresponding channel. It may be notifications, transactions, - // new blocks. For now only notifications. + current: newSubChannels(), + + subscribedEvents: make(map[util.Uint160]bool), + subscribedNotaryEvents: make(map[util.Uint160]bool), + } + // Worker listens all events from temporary NeoGo channel and puts them + // into corresponding permanent channels. go sub.routeNotifications(ctx) return sub, nil } +func (s *subscriber) routeNotifications(ctx context.Context) { + var ( + // TODO: not needed after nspcc-dev/neo-go#2980. + cliCh = s.client.NotificationChannel() + restoreCh = make(chan bool) + restoreInProgress bool + ) + +routeloop: + for { + var connLost bool + s.RLock() + curr := s.current + s.RUnlock() + select { + case <-ctx.Done(): + break routeloop + case ev, ok := <-curr.NotifyChan: + if ok { + s.client.Metrics().IncNotificationCount("notify") + s.notifyChan <- ev + } else { + connLost = true + } + case ev, ok := <-curr.BlockChan: + if ok { + s.client.Metrics().IncNotificationCount("block") + s.client.Metrics().SetLastBlock(ev.Index) + s.blockChan <- ev + } else { + connLost = true + } + case ev, ok := <-curr.NotaryChan: + if ok { + s.client.Metrics().IncNotificationCount("notary") + s.notaryChan <- ev + } else { + connLost = true + } + case _, ok := <-cliCh: + connLost = !ok + case ok := <-restoreCh: + restoreInProgress = false + if !ok { + connLost = true + } + } + if connLost { + if !restoreInProgress { + restoreInProgress, cliCh = s.switchEndpoint(ctx, restoreCh) + if !restoreInProgress { + break routeloop + } + curr.drain() + } else { // Avoid getting additional !ok events. + s.Lock() + s.current.NotifyChan = nil + s.current.BlockChan = nil + s.current.NotaryChan = nil + s.Unlock() + } + } + } + close(s.notifyChan) + close(s.blockChan) + close(s.notaryChan) +} + +func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) (bool, <-chan rpcclient.Notification) { + s.log.Info("RPC connection lost, attempting reconnect") + if !s.client.SwitchRPC(ctx) { + s.log.Error("can't switch RPC node") + return false, nil + } + + cliCh := s.client.NotificationChannel() + + s.Lock() + chs := newSubChannels() + go func() { + finishCh <- s.restoreSubscriptions(chs.NotifyChan, chs.BlockChan, chs.NotaryChan) + }() + s.current = chs + s.Unlock() + + s.client.Metrics().IncSwitchCount() + return true, cliCh +} + +func newSubChannels() subChannels { + return subChannels{ + NotifyChan: make(chan *state.ContainedNotificationEvent), + BlockChan: make(chan *block.Block), + NotaryChan: make(chan *result.NotaryRequestEvent), + } +} + +func (s *subChannels) drain() { +drainloop: + for { + select { + case _, ok := <-s.NotifyChan: + if !ok { + s.NotifyChan = nil + } + case _, ok := <-s.BlockChan: + if !ok { + s.BlockChan = nil + } + case _, ok := <-s.NotaryChan: + if !ok { + s.NotaryChan = nil + } + default: + break drainloop + } + } +} + +// restoreSubscriptions restores subscriptions according to +// cached information about them. +func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotificationEvent, + blCh chan<- *block.Block, notaryCh chan<- *result.NotaryRequestEvent) bool { + var err error + + // new block events restoration + if s.subscribedToNewBlocks { + _, err = s.client.ReceiveBlocks(blCh) + if err != nil { + s.log.Error(logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, zap.Error(err)) + return false + } + } + + // notification events restoration + for contract := range s.subscribedEvents { + contract := contract // See https://github.com/nspcc-dev/neo-go/issues/2890 + _, err = s.client.ReceiveExecutionNotifications(contract, notifCh) + if err != nil { + s.log.Error(logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) + return false + } + } + + // notary notification events restoration + for signer := range s.subscribedNotaryEvents { + signer := signer // See https://github.com/nspcc-dev/neo-go/issues/2890 + _, err = s.client.ReceiveNotaryRequests(signer, notaryCh) + if err != nil { + s.log.Error(logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) + return false + } + } + return true +} + // awaitHeight checks if remote client has least expected block height and // returns error if it is not reached that height after timeout duration. // This function is required to avoid connections to unsynced RPC nodes, because diff --git a/pkg/morph/timer/block.go b/pkg/morph/timer/block.go index 31c28e2ff..be20d3571 100644 --- a/pkg/morph/timer/block.go +++ b/pkg/morph/timer/block.go @@ -17,7 +17,7 @@ type BlockTickHandler func() type BlockTimer struct { rolledBack bool - mtx *sync.Mutex + mtx sync.Mutex dur BlockMeter @@ -64,7 +64,6 @@ func StaticBlockMeter(d uint32) BlockMeter { // Reset should be called before timer ticking. func NewBlockTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { return &BlockTimer{ - mtx: new(sync.Mutex), dur: dur, mul: 1, div: 1, @@ -80,7 +79,6 @@ func NewBlockTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { // Do not use delta handlers with pulse in this timer. func NewOneTickTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { return &BlockTimer{ - mtx: new(sync.Mutex), dur: dur, mul: 1, div: 1, diff --git a/pkg/network/address.go b/pkg/network/address.go index 020882980..8ad285725 100644 --- a/pkg/network/address.go +++ b/pkg/network/address.go @@ -47,7 +47,7 @@ func (a Address) URIAddr() string { panic(fmt.Errorf("could not get host addr: %w", err)) } - if !a.isTLSEnabled() { + if !a.IsTLSEnabled() { return host } diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index b8a0aa4bc..98d2f33e7 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -10,8 +10,11 @@ import ( rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -72,6 +75,17 @@ func (x *multiClient) createForAddress(ctx context.Context, addr network.Address prmInit.SetResponseInfoCallback(x.opts.ResponseCallback) } + prmDial.SetGRPCDialOptions( + grpc.WithChainUnaryInterceptor( + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), + ), + ) + c.Init(prmInit) err := c.Dial(ctx, prmDial) if err != nil { @@ -152,7 +166,7 @@ func (x *multiClient) iterateClients(ctx context.Context, f func(clientcore.Clie // non-status logic error that could be returned // from the SDK client; should not be considered // as a connection error - var siErr *object.SplitInfoError + var siErr *objectSDK.SplitInfoError success := err == nil || errors.Is(err, context.Canceled) || errors.As(err, &siErr) if success || firstErr == nil || errors.Is(firstErr, errRecentlyFailed) { @@ -181,7 +195,7 @@ func (x *multiClient) ReportError(err error) { // non-status logic error that could be returned // from the SDK client; should not be considered // as a connection error - var siErr *object.SplitInfoError + var siErr *objectSDK.SplitInfoError if errors.As(err, &siErr) { return } @@ -205,7 +219,7 @@ func (s *singleClient) invalidate() { s.Unlock() } -func (x *multiClient) ObjectPutInit(ctx context.Context, p client.PrmObjectPutInit) (res *client.ObjectWriter, err error) { +func (x *multiClient) ObjectPutInit(ctx context.Context, p client.PrmObjectPutInit) (res client.ObjectWriter, err error) { err = x.iterateClients(ctx, func(c clientcore.Client) error { res, err = c.ObjectPutInit(ctx, p) return err @@ -214,6 +228,15 @@ func (x *multiClient) ObjectPutInit(ctx context.Context, p client.PrmObjectPutIn return } +func (x *multiClient) ObjectPutSingle(ctx context.Context, p client.PrmObjectPutSingle) (res *client.ResObjectPutSingle, err error) { + err = x.iterateClients(ctx, func(c clientcore.Client) error { + res, err = c.ObjectPutSingle(ctx, p) + return err + }) + + return +} + func (x *multiClient) ContainerAnnounceUsedSpace(ctx context.Context, prm client.PrmAnnounceSpace) (res *client.ResAnnounceSpace, err error) { err = x.iterateClients(ctx, func(c clientcore.Client) error { res, err = c.ContainerAnnounceUsedSpace(ctx, prm) diff --git a/pkg/network/group.go b/pkg/network/group.go index c18feac27..a6de0653e 100644 --- a/pkg/network/group.go +++ b/pkg/network/group.go @@ -57,7 +57,7 @@ func (x AddressGroup) Len() int { // Less returns true if i-th address in AddressGroup supports TLS // and j-th one doesn't. func (x AddressGroup) Less(i, j int) bool { - return x[i].isTLSEnabled() && !x[j].isTLSEnabled() + return x[i].IsTLSEnabled() && !x[j].IsTLSEnabled() } // Swap swaps i-th and j-th addresses in AddressGroup. diff --git a/pkg/network/tls.go b/pkg/network/tls.go index de2c93694..9aac89c47 100644 --- a/pkg/network/tls.go +++ b/pkg/network/tls.go @@ -11,8 +11,8 @@ const ( // tls var is used for (un)wrapping other multiaddrs around TLS multiaddr. var tls, _ = multiaddr.NewMultiaddr("/" + tlsProtocolName) -// isTLSEnabled searches for wrapped TLS protocol in multiaddr. -func (a Address) isTLSEnabled() bool { +// IsTLSEnabled searches for wrapped TLS protocol in multiaddr. +func (a Address) IsTLSEnabled() bool { for _, protoc := range a.ma.Protocols() { if protoc.Code == multiaddr.P_TLS { return true diff --git a/pkg/network/tls_test.go b/pkg/network/tls_test.go index 25775eaf1..d93ea6a12 100644 --- a/pkg/network/tls_test.go +++ b/pkg/network/tls_test.go @@ -24,6 +24,6 @@ func TestAddress_TLSEnabled(t *testing.T) { err := addr.FromString(test.input) require.NoError(t, err) - require.Equal(t, test.wantTLS, addr.isTLSEnabled(), test.input) + require.Equal(t, test.wantTLS, addr.IsTLSEnabled(), test.input) } } diff --git a/pkg/network/transport/object/grpc/service.go b/pkg/network/transport/object/grpc/service.go index 7fa60f99c..7c6b395d5 100644 --- a/pkg/network/transport/object/grpc/service.go +++ b/pkg/network/transport/object/grpc/service.go @@ -110,3 +110,17 @@ func (s *Server) GetRangeHash(ctx context.Context, req *objectGRPC.GetRangeHashR return resp.ToGRPCMessage().(*objectGRPC.GetRangeHashResponse), nil } + +func (s *Server) PutSingle(ctx context.Context, req *objectGRPC.PutSingleRequest) (*objectGRPC.PutSingleResponse, error) { + putSingleReq := &object.PutSingleRequest{} + if err := putSingleReq.FromGRPCMessage(req); err != nil { + return nil, err + } + + resp, err := s.srv.PutSingle(ctx, putSingleReq) + if err != nil { + return nil, err + } + + return resp.ToGRPCMessage().(*objectGRPC.PutSingleResponse), nil +} diff --git a/pkg/services/accounting/executor.go b/pkg/services/accounting/executor.go index 402845957..b0722cf8a 100644 --- a/pkg/services/accounting/executor.go +++ b/pkg/services/accounting/executor.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" ) type ServiceExecutor interface { @@ -12,13 +13,15 @@ type ServiceExecutor interface { } type executorSvc struct { - exec ServiceExecutor + exec ServiceExecutor + respSvc *response.Service } // NewExecutionService wraps ServiceExecutor and returns Accounting Service interface. -func NewExecutionService(exec ServiceExecutor) Server { +func NewExecutionService(exec ServiceExecutor, respSvc *response.Service) Server { return &executorSvc{ - exec: exec, + exec: exec, + respSvc: respSvc, } } @@ -31,5 +34,6 @@ func (s *executorSvc) Balance(ctx context.Context, req *accounting.BalanceReques resp := new(accounting.BalanceResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/accounting/response.go b/pkg/services/accounting/response.go deleted file mode 100644 index a78ac6fd6..000000000 --- a/pkg/services/accounting/response.go +++ /dev/null @@ -1,37 +0,0 @@ -package accounting - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns accounting service instance that passes internal service -// call to response service. -func NewResponseService(accSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: accSvc, - } -} - -func (s *responseService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Balance(ctx, req.(*accounting.BalanceRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*accounting.BalanceResponse), nil -} diff --git a/pkg/services/accounting/sign.go b/pkg/services/accounting/sign.go index e98d9b3af..be7b08a39 100644 --- a/pkg/services/accounting/sign.go +++ b/pkg/services/accounting/sign.go @@ -22,17 +22,6 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { } func (s *signService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Balance(ctx, req.(*accounting.BalanceRequest)) - }, - func() util.ResponseMessage { - return new(accounting.BalanceResponse) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*accounting.BalanceResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Balance(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/container/executor.go b/pkg/services/container/executor.go index b4705d258..d4ae11d62 100644 --- a/pkg/services/container/executor.go +++ b/pkg/services/container/executor.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" ) type ServiceExecutor interface { @@ -21,12 +22,15 @@ type executorSvc struct { Server exec ServiceExecutor + + respSvc *response.Service } // NewExecutionService wraps ServiceExecutor and returns Container Service interface. -func NewExecutionService(exec ServiceExecutor) Server { +func NewExecutionService(exec ServiceExecutor, respSvc *response.Service) Server { return &executorSvc{ - exec: exec, + exec: exec, + respSvc: respSvc, } } @@ -44,6 +48,7 @@ func (s *executorSvc) Put(ctx context.Context, req *container.PutRequest) (*cont resp := new(container.PutResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -61,6 +66,7 @@ func (s *executorSvc) Delete(ctx context.Context, req *container.DeleteRequest) resp := new(container.DeleteResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -73,6 +79,7 @@ func (s *executorSvc) Get(ctx context.Context, req *container.GetRequest) (*cont resp := new(container.GetResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -85,6 +92,7 @@ func (s *executorSvc) List(ctx context.Context, req *container.ListRequest) (*co resp := new(container.ListResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -102,6 +110,7 @@ func (s *executorSvc) SetExtendedACL(ctx context.Context, req *container.SetExte resp := new(container.SetExtendedACLResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -114,5 +123,6 @@ func (s *executorSvc) GetExtendedACL(ctx context.Context, req *container.GetExte resp := new(container.GetExtendedACLResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index 8e6b30856..dec022219 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -52,7 +52,7 @@ func NewExecutor(rdr Reader, wrt Writer) containerSvc.ServiceExecutor { func (s *morphExecutor) Put(_ context.Context, tokV2 *sessionV2.Token, body *container.PutRequestBody) (*container.PutResponseBody, error) { sigV2 := body.GetSignature() if sigV2 == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return nil, errors.New("missing signature") } @@ -109,8 +109,6 @@ func (s *morphExecutor) Delete(_ context.Context, tokV2 *sessionV2.Token, body * return nil, fmt.Errorf("invalid container ID: %w", err) } - sig := body.GetSignature().GetSign() - var tok *session.Container if tokV2 != nil { @@ -124,9 +122,9 @@ func (s *morphExecutor) Delete(_ context.Context, tokV2 *sessionV2.Token, body * var rmWitness containercore.RemovalWitness - rmWitness.SetContainerID(id) - rmWitness.SetSignature(sig) - rmWitness.SetSessionToken(tok) + rmWitness.ContainerID = id + rmWitness.Signature = body.GetSignature() + rmWitness.SessionToken = tok err = s.wrt.Delete(rmWitness) if err != nil { @@ -208,7 +206,7 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) func (s *morphExecutor) SetExtendedACL(_ context.Context, tokV2 *sessionV2.Token, body *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) { sigV2 := body.GetSignature() if sigV2 == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return nil, errors.New("missing signature") } diff --git a/pkg/services/container/response.go b/pkg/services/container/response.go deleted file mode 100644 index 138974537..000000000 --- a/pkg/services/container/response.go +++ /dev/null @@ -1,115 +0,0 @@ -package container - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns container service instance that passes internal service -// call to response service. -func NewResponseService(cnrSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: cnrSvc, - } -} - -func (s *responseService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Put(ctx, req.(*container.PutRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.PutResponse), nil -} - -func (s *responseService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*container.DeleteRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.DeleteResponse), nil -} - -func (s *responseService) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Get(ctx, req.(*container.GetRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.GetResponse), nil -} - -func (s *responseService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.List(ctx, req.(*container.ListRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.ListResponse), nil -} - -func (s *responseService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.SetExtendedACL(ctx, req.(*container.SetExtendedACLRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.SetExtendedACLResponse), nil -} - -func (s *responseService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetExtendedACL(ctx, req.(*container.GetExtendedACLRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.GetExtendedACLResponse), nil -} - -func (s *responseService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.AnnounceUsedSpace(ctx, req.(*container.AnnounceUsedSpaceRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.AnnounceUsedSpaceResponse), nil -} diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index 9e77e2e21..b336f19c3 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -22,113 +22,64 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { } func (s *signService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Put(ctx, req.(*container.PutRequest)) - }, - func() util.ResponseMessage { - return new(container.PutResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.PutResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*container.PutResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Put(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*container.DeleteRequest)) - }, - func() util.ResponseMessage { - return new(container.DeleteResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.DeleteResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*container.DeleteResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Get(ctx, req.(*container.GetRequest)) - }, - func() util.ResponseMessage { - return new(container.GetResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.GetResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*container.GetResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Get(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.List(ctx, req.(*container.ListRequest)) - }, - func() util.ResponseMessage { - return new(container.ListResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.ListResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*container.ListResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.List(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.SetExtendedACL(ctx, req.(*container.SetExtendedACLRequest)) - }, - func() util.ResponseMessage { - return new(container.SetExtendedACLResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.SetExtendedACLResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*container.SetExtendedACLResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.SetExtendedACL(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetExtendedACL(ctx, req.(*container.GetExtendedACLRequest)) - }, - func() util.ResponseMessage { - return new(container.GetExtendedACLResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.GetExtendedACLResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*container.GetExtendedACLResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.GetExtendedACL(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.AnnounceUsedSpace(ctx, req.(*container.AnnounceUsedSpaceRequest)) - }, - func() util.ResponseMessage { - return new(container.AnnounceUsedSpaceResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.AnnounceUsedSpaceResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*container.AnnounceUsedSpaceResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.AnnounceUsedSpace(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/control/ir/rpc.go b/pkg/services/control/ir/rpc.go index 6b2234954..1b635c149 100644 --- a/pkg/services/control/ir/rpc.go +++ b/pkg/services/control/ir/rpc.go @@ -11,6 +11,7 @@ const serviceName = "ircontrol.ControlService" const ( rpcHealthCheck = "HealthCheck" rpcTickEpoch = "TickEpoch" + rpcRemoveNode = "RemoveNode" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -31,6 +32,14 @@ func TickEpoch( return sendUnary[TickEpochRequest, TickEpochResponse](cli, rpcTickEpoch, req, opts...) } +func RemoveNode( + cli *client.Client, + req *RemoveNodeRequest, + opts ...client.CallOption, +) (*RemoveNodeResponse, error) { + return sendUnary[RemoveNodeRequest, RemoveNodeResponse](cli, rpcRemoveNode, req, opts...) +} + func sendUnary[I, O grpc.Message](cli *client.Client, rpcName string, req *I, opts ...client.CallOption) (*O, error) { var resp O wResp := &responseWrapper[*O]{ diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 56e2e3f79..680d1e606 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -1,9 +1,11 @@ package control import ( + "bytes" "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -57,3 +59,43 @@ func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*c return resp, nil } + +// RemoveNode forces a node removal. +// +// If request is not signed with a key from white list, permission error returns. +func (s *Server) RemoveNode(_ context.Context, req *control.RemoveNodeRequest) (*control.RemoveNodeResponse, error) { + if err := s.isValidRequest(req); err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + resp := new(control.RemoveNodeResponse) + resp.SetBody(new(control.RemoveNodeResponse_Body)) + + nm, err := s.netmapClient.NetMap() + if err != nil { + return nil, fmt.Errorf("getting netmap: %w", err) + } + var nodeInfo netmap.NodeInfo + for _, info := range nm.Nodes() { + if bytes.Equal(info.PublicKey(), req.GetBody().GetKey()) { + nodeInfo = info + break + } + } + if len(nodeInfo.PublicKey()) == 0 { + return nil, status.Error(codes.NotFound, "no such node") + } + if nodeInfo.IsOffline() { + return nil, status.Error(codes.FailedPrecondition, "node is already offline") + } + + if err := s.netmapClient.ForceRemovePeer(nodeInfo); err != nil { + return nil, fmt.Errorf("forcing node removal: %w", err) + } + + if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return resp, nil +} diff --git a/pkg/services/control/ir/server/sign.go b/pkg/services/control/ir/server/sign.go index 4ada98468..f72d51f9e 100644 --- a/pkg/services/control/ir/server/sign.go +++ b/pkg/services/control/ir/server/sign.go @@ -24,7 +24,7 @@ var errDisallowedKey = errors.New("key is not in the allowed list") func (s *Server) isValidRequest(req SignedMessage) error { sign := req.GetSignature() if sign == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("missing signature") } @@ -50,7 +50,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { return fmt.Errorf("marshal request body: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetKey(sign.GetKey()) sigV2.SetSign(sign.GetSign()) @@ -62,7 +62,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { } if !sig.Verify(binBody) { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("invalid signature") } @@ -83,7 +83,7 @@ func SignMessage(key *ecdsa.PrivateKey, msg SignedMessage) error { return fmt.Errorf("calculate signature: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sig.WriteToV2(&sigV2) diff --git a/pkg/services/control/ir/service.go b/pkg/services/control/ir/service.go index 1aaec2c87..b2db2b43a 100644 --- a/pkg/services/control/ir/service.go +++ b/pkg/services/control/ir/service.go @@ -32,3 +32,15 @@ func (x *TickEpochResponse) SetBody(v *TickEpochResponse_Body) { x.Body = v } } + +func (x *RemoveNodeRequest) SetBody(v *RemoveNodeRequest_Body) { + if x != nil { + x.Body = v + } +} + +func (x *RemoveNodeResponse) SetBody(v *RemoveNodeResponse_Body) { + if x != nil { + x.Body = v + } +} diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index 84acdfc82..bec74a3be 100644 Binary files a/pkg/services/control/ir/service.pb.go and b/pkg/services/control/ir/service.pb.go differ diff --git a/pkg/services/control/ir/service.proto b/pkg/services/control/ir/service.proto index 5862e8fbd..d647db0df 100644 --- a/pkg/services/control/ir/service.proto +++ b/pkg/services/control/ir/service.proto @@ -12,6 +12,8 @@ service ControlService { rpc HealthCheck (HealthCheckRequest) returns (HealthCheckResponse); // Forces a new epoch to be signaled by the IR node with high probability. rpc TickEpoch (TickEpochRequest) returns (TickEpochResponse); + // Forces a node removal to be signaled by the IR node with high probability. + rpc RemoveNode (RemoveNodeRequest) returns (RemoveNodeResponse); } // Health check request. @@ -57,3 +59,19 @@ message TickEpochResponse { Body body = 1; Signature signature = 2; } + +message RemoveNodeRequest { + message Body{ + bytes key = 1; + } + + Body body = 1; + Signature signature = 2; +} + +message RemoveNodeResponse { + message Body{} + + Body body = 1; + Signature signature = 2; +} diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index d480f0b50..c93253105 100644 Binary files a/pkg/services/control/ir/service_frostfs.pb.go and b/pkg/services/control/ir/service_frostfs.pb.go differ diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index 700d340ca..6ba214da0 100644 Binary files a/pkg/services/control/ir/service_grpc.pb.go and b/pkg/services/control/ir/service_grpc.pb.go differ diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 31ebfa760..a2e7c411a 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -8,15 +8,18 @@ import ( const serviceName = "control.ControlService" const ( - rpcHealthCheck = "HealthCheck" - rpcSetNetmapStatus = "SetNetmapStatus" - rpcDropObjects = "DropObjects" - rpcListShards = "ListShards" - rpcSetShardMode = "SetShardMode" - rpcSynchronizeTree = "SynchronizeTree" - rpcEvacuateShard = "EvacuateShard" - rpcFlushCache = "FlushCache" - rpcDoctor = "Doctor" + rpcHealthCheck = "HealthCheck" + rpcSetNetmapStatus = "SetNetmapStatus" + rpcDropObjects = "DropObjects" + rpcListShards = "ListShards" + rpcSetShardMode = "SetShardMode" + rpcSynchronizeTree = "SynchronizeTree" + rpcEvacuateShard = "EvacuateShard" + rpcStartShardEvacuation = "StartShardEvacuation" + rpcGetShardEvacuationStatus = "GetShardEvacuationStatus" + rpcStopShardEvacuation = "StopShardEvacuation" + rpcFlushCache = "FlushCache" + rpcDoctor = "Doctor" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -141,6 +144,45 @@ func EvacuateShard(cli *client.Client, req *EvacuateShardRequest, opts ...client return wResp.message, nil } +// StartShardEvacuation executes ControlService.StartShardEvacuation RPC. +func StartShardEvacuation(cli *client.Client, req *StartShardEvacuationRequest, opts ...client.CallOption) (*StartShardEvacuationResponse, error) { + wResp := newResponseWrapper[StartShardEvacuationResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStartShardEvacuation), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + +// GetShardEvacuationStatus executes ControlService.GetShardEvacuationStatus RPC. +func GetShardEvacuationStatus(cli *client.Client, req *GetShardEvacuationStatusRequest, opts ...client.CallOption) (*GetShardEvacuationStatusResponse, error) { + wResp := newResponseWrapper[GetShardEvacuationStatusResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcGetShardEvacuationStatus), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + +// StopShardEvacuation executes ControlService.StopShardEvacuation RPC. +func StopShardEvacuation(cli *client.Client, req *StopShardEvacuationRequest, opts ...client.CallOption) (*StopShardEvacuationResponse, error) { + wResp := newResponseWrapper[StopShardEvacuationResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStopShardEvacuation), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + // FlushCache executes ControlService.FlushCache RPC. func FlushCache(cli *client.Client, req *FlushCacheRequest, opts ...client.CallOption) (*FlushCacheResponse, error) { wResp := newResponseWrapper[FlushCacheResponse]() diff --git a/pkg/services/control/server/convert.go b/pkg/services/control/server/convert.go new file mode 100644 index 000000000..1d29ed406 --- /dev/null +++ b/pkg/services/control/server/convert.go @@ -0,0 +1,60 @@ +package control + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "github.com/mr-tron/base58" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func stateToResponse(state *engine.EvacuationState) (*control.GetShardEvacuationStatusResponse, error) { + shardIDs := make([][]byte, 0, len(state.ShardIDs())) + for _, shID := range state.ShardIDs() { + id, err := base58.Decode(shID) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("invalid shard id format: %s", shID)) + } + shardIDs = append(shardIDs, id) + } + var evacStatus control.GetShardEvacuationStatusResponse_Body_Status + switch state.ProcessingStatus() { + case engine.EvacuateProcessStateRunning: + evacStatus = control.GetShardEvacuationStatusResponse_Body_RUNNING + case engine.EvacuateProcessStateCompleted: + evacStatus = control.GetShardEvacuationStatusResponse_Body_COMPLETED + default: + evacStatus = control.GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED + } + var startedAt *control.GetShardEvacuationStatusResponse_Body_UnixTimestamp + if state.StartedAt() != nil { + startedAt = &control.GetShardEvacuationStatusResponse_Body_UnixTimestamp{ + Value: state.StartedAt().Unix(), + } + } + var duration *control.GetShardEvacuationStatusResponse_Body_Duration + if state.StartedAt() != nil { + end := time.Now().UTC() + if state.FinishedAt() != nil { + end = *state.FinishedAt() + } + duration = &control.GetShardEvacuationStatusResponse_Body_Duration{ + Seconds: int64(end.Sub(*state.StartedAt()).Seconds()), + } + } + return &control.GetShardEvacuationStatusResponse{ + Body: &control.GetShardEvacuationStatusResponse_Body{ + Shard_ID: shardIDs, + Evacuated: state.Evacuated(), + Total: state.Total(), + Failed: state.Failed(), + Status: evacStatus, + StartedAt: startedAt, + Duration: duration, + ErrorMessage: state.ErrorMessage(), + }, + }, nil +} diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index afa4011b9..8f62c3489 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -37,7 +37,7 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe resp := &control.EvacuateShardResponse{ Body: &control.EvacuateShardResponse_Body{ - Count: uint32(res.Count()), + Count: uint32(res.Evacuated()), }, } @@ -84,11 +84,12 @@ func (s *Server) replicate(ctx context.Context, addr oid.Address, obj *objectSDK } var res replicatorResult - var task replicator.Task - task.SetObject(obj) - task.SetObjectAddress(addr) - task.SetCopiesNumber(1) - task.SetNodes(nodes) + task := replicator.Task{ + NumCopies: 1, + Addr: addr, + Obj: obj, + Nodes: nodes, + } s.replicator.HandleTask(ctx, task, &res) if res.count == 0 { diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go new file mode 100644 index 000000000..cdf3656e2 --- /dev/null +++ b/pkg/services/control/server/evacuate_async.go @@ -0,0 +1,97 @@ +package control + +import ( + "context" + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartShardEvacuationRequest) (*control.StartShardEvacuationResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + var prm engine.EvacuateShardPrm + prm.WithShardIDList(s.getShardIDList(req.GetBody().GetShard_ID())) + prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) + prm.WithFaultHandler(s.replicate) + prm.WithAsync(true) + + _, err = s.s.Evacuate(ctx, prm) + if err != nil { + var logicalErr logicerr.Logical + if errors.As(err, &logicalErr) { + return nil, status.Error(codes.Aborted, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.StartShardEvacuationResponse{ + Body: &control.StartShardEvacuationResponse_Body{}, + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} + +func (s *Server) GetShardEvacuationStatus(ctx context.Context, req *control.GetShardEvacuationStatusRequest) (*control.GetShardEvacuationStatusResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + state, err := s.s.GetEvacuationState(ctx) + if err != nil { + var logicalErr logicerr.Logical + if errors.As(err, &logicalErr) { + return nil, status.Error(codes.Aborted, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp, err := stateToResponse(state) + if err != nil { + return nil, err + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} + +func (s *Server) StopShardEvacuation(ctx context.Context, req *control.StopShardEvacuationRequest) (*control.StopShardEvacuationResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + err = s.s.EnqueRunningEvacuationStop(ctx) + if err != nil { + var logicalErr logicerr.Logical + if errors.As(err, &logicalErr) { + return nil, status.Error(codes.Aborted, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.StopShardEvacuationResponse{ + Body: &control.StopShardEvacuationResponse_Body{}, + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} diff --git a/pkg/services/control/server/sign.go b/pkg/services/control/server/sign.go index 726cdf341..acc405821 100644 --- a/pkg/services/control/server/sign.go +++ b/pkg/services/control/server/sign.go @@ -24,7 +24,7 @@ var errDisallowedKey = errors.New("key is not in the allowed list") func (s *Server) isValidRequest(req SignedMessage) error { sign := req.GetSignature() if sign == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("missing signature") } @@ -50,7 +50,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { return fmt.Errorf("marshal request body: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetKey(sign.GetKey()) sigV2.SetSign(sign.GetSign()) @@ -62,7 +62,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { } if !sig.Verify(binBody) { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("invalid signature") } @@ -83,7 +83,7 @@ func SignMessage(key *ecdsa.PrivateKey, msg SignedMessage) error { return fmt.Errorf("calculate signature: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sig.WriteToV2(&sigV2) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index a126ce16d..b1bebb1e2 100644 Binary files a/pkg/services/control/service.pb.go and b/pkg/services/control/service.pb.go differ diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 32a87c744..a80deb2da 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -27,8 +27,18 @@ service ControlService { rpc SynchronizeTree (SynchronizeTreeRequest) returns (SynchronizeTreeResponse); // EvacuateShard moves all data from one shard to the others. + // Deprecated: Use StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation rpc EvacuateShard (EvacuateShardRequest) returns (EvacuateShardResponse); + // StartShardEvacuation starts moving all data from one shard to the others. + rpc StartShardEvacuation (StartShardEvacuationRequest) returns (StartShardEvacuationResponse); + + // GetShardEvacuationStatus returns evacuation status. + rpc GetShardEvacuationStatus (GetShardEvacuationStatusRequest) returns (GetShardEvacuationStatusResponse); + + // StopShardEvacuation stops moving all data from one shard to the others. + rpc StopShardEvacuation (StopShardEvacuationRequest) returns (StopShardEvacuationResponse); + // FlushCache moves all data from one shard to the others. rpc FlushCache (FlushCacheRequest) returns (FlushCacheResponse); @@ -298,3 +308,97 @@ message DoctorResponse { Body body = 1; Signature signature = 2; } + +// StartShardEvacuation request. +message StartShardEvacuationRequest { + // Request body structure. + message Body { + // IDs of the shards. + repeated bytes shard_ID = 1; + // Flag indicating whether object read errors should be ignored. + bool ignore_errors = 2; + } + + Body body = 1; + Signature signature = 2; +} + +// StartShardEvacuation response. +message StartShardEvacuationResponse { + // Response body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} + +// GetShardEvacuationStatus request. +message GetShardEvacuationStatusRequest { + // Request body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} + +// GetShardEvacuationStatus response. +message GetShardEvacuationStatusResponse { + // Response body structure. + message Body { + // Evacuate status enum. + enum Status { + EVACUATE_SHARD_STATUS_UNDEFINED = 0; + RUNNING = 1; + COMPLETED = 2; + } + + // Unix timestamp value. + message UnixTimestamp { + int64 value = 1; + } + + // Duration in seconds. + message Duration { + int64 seconds = 1; + } + + // Total objects to evacuate count. The value is approximate, so evacuated + failed == total is not guaranteed after completion. + uint64 total = 1; + // Evacuated objects count. + uint64 evacuated = 2; + // Failed objects count. + uint64 failed = 3; + + // Shard IDs. + repeated bytes shard_ID = 4; + // Evacuation process status. + Status status = 5; + // Evacuation process duration. + Duration duration = 6; + // Evacuation process started at timestamp. + UnixTimestamp started_at = 7; + // Error message if evacuation failed. + string error_message = 8; + } + + Body body = 1; + Signature signature = 2; +} + +// StopShardEvacuation request. +message StopShardEvacuationRequest { + // Request body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} + +// StopShardEvacuation response. +message StopShardEvacuationResponse { + // Response body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index b9b865a90..979c22685 100644 Binary files a/pkg/services/control/service_frostfs.pb.go and b/pkg/services/control/service_frostfs.pb.go differ diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 3fa1e54de..8afc6086a 100644 Binary files a/pkg/services/control/service_grpc.pb.go and b/pkg/services/control/service_grpc.pb.go differ diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index 9d6864759..d2ee50770 100644 Binary files a/pkg/services/control/types.pb.go and b/pkg/services/control/types.pb.go differ diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index d77c69d4d..d1e7a949e 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/version" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" versionsdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" ) @@ -18,6 +19,8 @@ type executorSvc struct { state NodeState netInfo NetworkInfo + + respSvc *response.Service } // NodeState encapsulates information @@ -42,8 +45,8 @@ type NetworkInfo interface { Dump(versionsdk.Version) (*netmapSDK.NetworkInfo, error) } -func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo) Server { - if s == nil || netInfo == nil || !version.IsValid(v) { +func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo, respSvc *response.Service) Server { + if s == nil || netInfo == nil || !version.IsValid(v) || respSvc == nil { // this should never happen, otherwise it programmers bug panic("can't create netmap execution service") } @@ -51,6 +54,7 @@ func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo) res := &executorSvc{ state: s, netInfo: netInfo, + respSvc: respSvc, } v.WriteToV2(&res.version) @@ -96,6 +100,7 @@ func (s *executorSvc) LocalNodeInfo( resp := new(netmap.LocalNodeInfoResponse) resp.SetBody(body) + s.respSvc.SetMeta(resp) return resp, nil } @@ -126,6 +131,7 @@ func (s *executorSvc) NetworkInfo( resp := new(netmap.NetworkInfoResponse) resp.SetBody(body) + s.respSvc.SetMeta(resp) return resp, nil } @@ -143,5 +149,6 @@ func (s *executorSvc) Snapshot(_ context.Context, _ *netmap.SnapshotRequest) (*n resp := new(netmap.SnapshotResponse) resp.SetBody(body) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/netmap/response.go b/pkg/services/netmap/response.go deleted file mode 100644 index 8b035e461..000000000 --- a/pkg/services/netmap/response.go +++ /dev/null @@ -1,63 +0,0 @@ -package netmap - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns netmap service instance that passes internal service -// call to response service. -func NewResponseService(nmSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: nmSvc, - } -} - -func (s *responseService) LocalNodeInfo(ctx context.Context, req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.LocalNodeInfo(ctx, req.(*netmap.LocalNodeInfoRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*netmap.LocalNodeInfoResponse), nil -} - -func (s *responseService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.NetworkInfo(ctx, req.(*netmap.NetworkInfoRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*netmap.NetworkInfoResponse), nil -} - -func (s *responseService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Snapshot(ctx, req.(*netmap.SnapshotRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*netmap.SnapshotResponse), nil -} diff --git a/pkg/services/netmap/sign.go b/pkg/services/netmap/sign.go index 85b19d862..2d01164a3 100644 --- a/pkg/services/netmap/sign.go +++ b/pkg/services/netmap/sign.go @@ -24,49 +24,28 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) LocalNodeInfo( ctx context.Context, req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.LocalNodeInfo(ctx, req.(*netmap.LocalNodeInfoRequest)) - }, - func() util.ResponseMessage { - return new(netmap.LocalNodeInfoResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(netmap.LocalNodeInfoResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*netmap.LocalNodeInfoResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.LocalNodeInfo(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.NetworkInfo(ctx, req.(*netmap.NetworkInfoRequest)) - }, - func() util.ResponseMessage { - return new(netmap.NetworkInfoResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(netmap.NetworkInfoResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*netmap.NetworkInfoResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.NetworkInfo(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Snapshot(ctx, req.(*netmap.SnapshotRequest)) - }, - func() util.ResponseMessage { - return new(netmap.SnapshotResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(netmap.SnapshotResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*netmap.SnapshotResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Snapshot(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/notificator/nats/service.go b/pkg/services/notificator/nats/service.go index 6a7e80a53..19f0cbde1 100644 --- a/pkg/services/notificator/nats/service.go +++ b/pkg/services/notificator/nats/service.go @@ -25,7 +25,7 @@ type Writer struct { js nats.JetStreamContext nc *nats.Conn - m *sync.RWMutex + m sync.RWMutex createdStreams map[string]struct{} opts } @@ -84,7 +84,6 @@ func (n *Writer) Notify(topic string, address oid.Address) error { // New creates new Writer. func New(oo ...Option) *Writer { w := &Writer{ - m: &sync.RWMutex{}, createdStreams: make(map[string]struct{}), opts: opts{ log: &logger.Logger{Logger: zap.L()}, diff --git a/pkg/services/object/acl/acl.go b/pkg/services/object/acl/acl.go index 351b4ad3b..921545c8b 100644 --- a/pkg/services/object/acl/acl.go +++ b/pkg/services/object/acl/acl.go @@ -1,10 +1,12 @@ package acl import ( + "context" "crypto/ecdsa" "crypto/elliptic" "errors" "fmt" + "io" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -17,39 +19,12 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) -// CheckerPrm groups parameters for Checker -// constructor. -type CheckerPrm struct { - eaclSrc container.EACLSource - validator *eaclSDK.Validator - localStorage *engine.StorageEngine - state netmap.State -} - -func (c *CheckerPrm) SetEACLSource(v container.EACLSource) *CheckerPrm { - c.eaclSrc = v - return c -} - -func (c *CheckerPrm) SetValidator(v *eaclSDK.Validator) *CheckerPrm { - c.validator = v - return c -} - -func (c *CheckerPrm) SetLocalStorage(v *engine.StorageEngine) *CheckerPrm { - c.localStorage = v - return c -} - -func (c *CheckerPrm) SetNetmapState(v netmap.State) *CheckerPrm { - c.state = v - return c -} - // Checker implements v2.ACLChecker interfaces and provides // ACL/eACL validation functionality. type Checker struct { @@ -59,6 +34,18 @@ type Checker struct { state netmap.State } +type localStorage struct { + ls *engine.StorageEngine +} + +func (s *localStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { + if s.ls == nil { + return nil, io.ErrUnexpectedEOF + } + + return engine.Head(ctx, s.ls, addr) +} + // Various EACL check errors. var ( errEACLDeniedByRule = errors.New("denied by rule") @@ -71,23 +58,17 @@ var ( // NewChecker creates Checker. // Panics if at least one of the parameter is nil. -func NewChecker(prm *CheckerPrm) *Checker { - panicOnNil := func(fieldName string, field any) { - if field == nil { - panic(fmt.Sprintf("incorrect field %s (%T): %v", fieldName, field, field)) - } - } - - panicOnNil("EACLSource", prm.eaclSrc) - panicOnNil("EACLValidator", prm.validator) - panicOnNil("LocalStorageEngine", prm.localStorage) - panicOnNil("NetmapState", prm.state) - +func NewChecker( + state netmap.State, + eaclSrc container.EACLSource, + validator *eaclSDK.Validator, + localStorage *engine.StorageEngine, +) *Checker { return &Checker{ - eaclSrc: prm.eaclSrc, - validator: prm.validator, - localStorage: prm.localStorage, - state: prm.state, + eaclSrc: eaclSrc, + validator: validator, + localStorage: localStorage, + state: state, } } @@ -193,26 +174,14 @@ func getRole(reqInfo v2.RequestInfo) eaclSDK.Role { } func (c *Checker) getHeaderSource(cnr cid.ID, msg any, reqInfo v2.RequestInfo) (eaclSDK.TypedHeaderSource, error) { - hdrSrcOpts := make([]eaclV2.Option, 0, 3) - - hdrSrcOpts = append(hdrSrcOpts, - eaclV2.WithLocalObjectStorage(c.localStorage), - eaclV2.WithCID(cnr), - eaclV2.WithOID(reqInfo.ObjectID()), - ) - + var xHeaderSource eaclV2.XHeaderSource if req, ok := msg.(eaclV2.Request); ok { - hdrSrcOpts = append(hdrSrcOpts, eaclV2.WithServiceRequest(req)) + xHeaderSource = eaclV2.NewRequestXHeaderSource(req) } else { - hdrSrcOpts = append(hdrSrcOpts, - eaclV2.WithServiceResponse( - msg.(eaclV2.Response), - reqInfo.Request().(eaclV2.Request), - ), - ) + xHeaderSource = eaclV2.NewResponseXHeaderSource(msg.(eaclV2.Response), reqInfo.Request().(eaclV2.Request)) } - hdrSrc, err := eaclV2.NewMessageHeaderSource(hdrSrcOpts...) + hdrSrc, err := eaclV2.NewMessageHeaderSource(&localStorage{ls: c.localStorage}, xHeaderSource, cnr, eaclV2.WithOID(reqInfo.ObjectID())) if err != nil { return nil, fmt.Errorf("can't parse headers: %w", err) } diff --git a/pkg/services/object/acl/acl_test.go b/pkg/services/object/acl/acl_test.go index d3ad1e6fd..b9b82dac8 100644 --- a/pkg/services/object/acl/acl_test.go +++ b/pkg/services/object/acl/acl_test.go @@ -27,12 +27,11 @@ func (e emptyNetmapState) CurrentEpoch() uint64 { } func TestStickyCheck(t *testing.T) { - checker := NewChecker(new(CheckerPrm). - SetLocalStorage(&engine.StorageEngine{}). - SetValidator(eaclSDK.NewValidator()). - SetEACLSource(emptyEACLSource{}). - SetNetmapState(emptyNetmapState{}), - ) + checker := NewChecker( + emptyNetmapState{}, + emptyEACLSource{}, + eaclSDK.NewValidator(), + &engine.StorageEngine{}) t.Run("system role", func(t *testing.T) { var info v2.RequestInfo diff --git a/pkg/services/object/acl/eacl/v2/eacl_test.go b/pkg/services/object/acl/eacl/v2/eacl_test.go index ce5d98d5d..023b99239 100644 --- a/pkg/services/object/acl/eacl/v2/eacl_test.go +++ b/pkg/services/object/acl/eacl/v2/eacl_test.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -22,12 +22,12 @@ type testLocalStorage struct { expAddr oid.Address - obj *object.Object + obj *objectSDK.Object err error } -func (s *testLocalStorage) Head(ctx context.Context, addr oid.Address) (*object.Object, error) { +func (s *testLocalStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { require.True(s.t, addr.Container().Equals(s.expAddr.Container())) require.True(s.t, addr.Object().Equals(s.expAddr.Object())) @@ -69,11 +69,11 @@ func TestHeadRequest(t *testing.T) { meta.SetXHeaders(xHdrs) - obj := object.New() + obj := objectSDK.New() attrKey := "attr_key" attrVal := "attr_val" - var attr object.Attribute + var attr objectSDK.Attribute attr.SetKey(attrKey) attr.SetValue(attrVal) obj.SetAttributes(attr) @@ -103,9 +103,9 @@ func TestHeadRequest(t *testing.T) { newSource := func(t *testing.T) eaclSDK.TypedHeaderSource { hdrSrc, err := NewMessageHeaderSource( - WithObjectStorage(lStorage), - WithServiceRequest(req), - WithCID(addr.Container()), + lStorage, + NewRequestXHeaderSource(req), + addr.Container(), WithOID(&id)) require.NoError(t, err) return hdrSrc diff --git a/pkg/services/object/acl/eacl/v2/headers.go b/pkg/services/object/acl/eacl/v2/headers.go index 095810848..7408c96e9 100644 --- a/pkg/services/object/acl/eacl/v2/headers.go +++ b/pkg/services/object/acl/eacl/v2/headers.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -21,14 +21,14 @@ type Option func(*cfg) type cfg struct { storage ObjectStorage - msg xHeaderSource + msg XHeaderSource cnr cid.ID obj *oid.ID } type ObjectStorage interface { - Head(context.Context, oid.Address) (*object.Object, error) + Head(context.Context, oid.Address) (*objectSDK.Object, error) } type Request interface { @@ -46,14 +46,12 @@ type headerSource struct { incompleteObjectHeaders bool } -func defaultCfg() *cfg { - return &cfg{ - storage: new(localStorage), +func NewMessageHeaderSource(os ObjectStorage, xhs XHeaderSource, cnrID cid.ID, opts ...Option) (eaclSDK.TypedHeaderSource, error) { + cfg := &cfg{ + storage: os, + cnr: cnrID, + msg: xhs, } -} - -func NewMessageHeaderSource(opts ...Option) (eaclSDK.TypedHeaderSource, error) { - cfg := defaultCfg() for i := range opts { opts[i](cfg) @@ -70,7 +68,7 @@ func NewMessageHeaderSource(opts ...Option) (eaclSDK.TypedHeaderSource, error) { return nil, err } - res.requestHeaders = requestHeaders(cfg.msg) + res.requestHeaders = cfg.msg.GetXHeaders() return res, nil } @@ -96,10 +94,6 @@ func (x xHeader) Value() string { return (*session.XHeader)(&x).GetValue() } -func requestHeaders(msg xHeaderSource) []eaclSDK.Header { - return msg.GetXHeaders() -} - var errMissingOID = errors.New("object ID is missing") func (h *cfg) readObjectHeaders(dst *headerSource) error { @@ -141,7 +135,7 @@ func (h *cfg) readObjectHeadersFromRequestXHeaderSource(m requestXHeaderSource, oV2.SetObjectID(v.GetObjectID()) oV2.SetHeader(v.GetHeader()) - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) } case *objectV2.SearchRequest: cnrV2 := req.GetBody().GetContainerID() @@ -171,7 +165,7 @@ func (h *cfg) readObjectHeadersResponseXHeaderSource(m responseXHeaderSource, ds oV2.SetObjectID(v.GetObjectID()) oV2.SetHeader(v.GetHeader()) - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) } case *objectV2.HeadResponse: oV2 := new(objectV2.Object) @@ -197,7 +191,7 @@ func (h *cfg) readObjectHeadersResponseXHeaderSource(m responseXHeaderSource, ds oV2.SetHeader(hdr) - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) } return nil } diff --git a/pkg/services/object/acl/eacl/v2/localstore.go b/pkg/services/object/acl/eacl/v2/localstore.go deleted file mode 100644 index 0f23e9881..000000000 --- a/pkg/services/object/acl/eacl/v2/localstore.go +++ /dev/null @@ -1,22 +0,0 @@ -package v2 - -import ( - "context" - "io" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -type localStorage struct { - ls *engine.StorageEngine -} - -func (s *localStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { - if s.ls == nil { - return nil, io.ErrUnexpectedEOF - } - - return engine.Head(ctx, s.ls, addr) -} diff --git a/pkg/services/object/acl/eacl/v2/object.go b/pkg/services/object/acl/eacl/v2/object.go index 0a63981cb..690e4aa70 100644 --- a/pkg/services/object/acl/eacl/v2/object.go +++ b/pkg/services/object/acl/eacl/v2/object.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -26,7 +26,7 @@ func u64Value(v uint64) string { return strconv.FormatUint(v, 10) } -func headersFromObject(obj *object.Object, cnr cid.ID, oid *oid.ID) []eaclSDK.Header { +func headersFromObject(obj *objectSDK.Object, cnr cid.ID, oid *oid.ID) []eaclSDK.Header { var count int for obj := obj; obj != nil; obj = obj.Parent() { count += 9 + len(obj.Attributes()) diff --git a/pkg/services/object/acl/eacl/v2/opts.go b/pkg/services/object/acl/eacl/v2/opts.go index 7657e8780..d91a21c75 100644 --- a/pkg/services/object/acl/eacl/v2/opts.go +++ b/pkg/services/object/acl/eacl/v2/opts.go @@ -1,48 +1,9 @@ package v2 import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -func WithObjectStorage(v ObjectStorage) Option { - return func(c *cfg) { - c.storage = v - } -} - -func WithLocalObjectStorage(v *engine.StorageEngine) Option { - return func(c *cfg) { - c.storage = &localStorage{ - ls: v, - } - } -} - -func WithServiceRequest(v Request) Option { - return func(c *cfg) { - c.msg = requestXHeaderSource{ - req: v, - } - } -} - -func WithServiceResponse(resp Response, req Request) Option { - return func(c *cfg) { - c.msg = responseXHeaderSource{ - resp: resp, - req: req, - } - } -} - -func WithCID(v cid.ID) Option { - return func(c *cfg) { - c.cnr = v - } -} - func WithOID(v *oid.ID) Option { return func(c *cfg) { c.obj = v diff --git a/pkg/services/object/acl/eacl/v2/xheader.go b/pkg/services/object/acl/eacl/v2/xheader.go index 246714af7..c1fdea9d8 100644 --- a/pkg/services/object/acl/eacl/v2/xheader.go +++ b/pkg/services/object/acl/eacl/v2/xheader.go @@ -5,7 +5,7 @@ import ( eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" ) -type xHeaderSource interface { +type XHeaderSource interface { GetXHeaders() []eaclSDK.Header } @@ -13,12 +13,20 @@ type requestXHeaderSource struct { req Request } +func NewRequestXHeaderSource(req Request) XHeaderSource { + return requestXHeaderSource{req: req} +} + type responseXHeaderSource struct { resp Response req Request } +func NewResponseXHeaderSource(resp Response, req Request) XHeaderSource { + return responseXHeaderSource{resp: resp, req: req} +} + func (s requestXHeaderSource) GetXHeaders() []eaclSDK.Header { ln := 0 diff --git a/pkg/services/object/acl/v2/opts.go b/pkg/services/object/acl/v2/opts.go index 7e937da06..15fcce884 100644 --- a/pkg/services/object/acl/v2/opts.go +++ b/pkg/services/object/acl/v2/opts.go @@ -1,9 +1,6 @@ package v2 import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" ) @@ -13,39 +10,3 @@ func WithLogger(v *logger.Logger) Option { c.log = v } } - -// WithNetmapSource return option to set -// netmap source. -func WithNetmapSource(v netmap.Source) Option { - return func(c *cfg) { - c.nm = v - } -} - -// WithContainerSource returns option to set container source. -func WithContainerSource(v container.Source) Option { - return func(c *cfg) { - c.containers = v - } -} - -// WithNextService returns option to set next object service. -func WithNextService(v objectSvc.ServiceServer) Option { - return func(c *cfg) { - c.next = v - } -} - -// WithEACLChecker returns option to set eACL checker. -func WithEACLChecker(v ACLChecker) Option { - return func(c *cfg) { - c.checker = v - } -} - -// WithIRFetcher returns option to set inner ring fetcher. -func WithIRFetcher(v InnerRingFetcher) Option { - return func(c *cfg) { - c.irFetcher = v - } -} diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 6544d78d7..412708b1c 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -6,6 +6,7 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" @@ -72,32 +73,26 @@ type cfg struct { next object.ServiceServer } -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - } -} - // New is a constructor for object ACL checking service. -func New(opts ...Option) Service { - cfg := defaultCfg() +func New(next object.ServiceServer, + nm netmap.Source, + irf InnerRingFetcher, + acl ACLChecker, + cs container.Source, + opts ...Option) Service { + cfg := &cfg{ + log: &logger.Logger{Logger: zap.L()}, + next: next, + nm: nm, + irFetcher: irf, + checker: acl, + containers: cs, + } for i := range opts { opts[i](cfg) } - panicOnNil := func(v any, name string) { - if v == nil { - panic(fmt.Sprintf("ACL service: %s is nil", name)) - } - } - - panicOnNil(cfg.next, "next Service") - panicOnNil(cfg.nm, "netmap client") - panicOnNil(cfg.irFetcher, "inner Ring fetcher") - panicOnNil(cfg.checker, "acl checker") - panicOnNil(cfg.containers, "container source") - return Service{ cfg: cfg, c: senderClassifier{ @@ -443,6 +438,65 @@ func (b Service) GetRangeHash( return b.next.GetRangeHash(ctx, request) } +func (b Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { + cnr, err := getContainerIDFromRequest(request) + if err != nil { + return nil, err + } + + idV2 := request.GetBody().GetObject().GetHeader().GetOwnerID() + if idV2 == nil { + return nil, errors.New("missing object owner") + } + + var idOwner user.ID + + err = idOwner.ReadFromV2(*idV2) + if err != nil { + return nil, fmt.Errorf("invalid object owner: %w", err) + } + + obj, err := getObjectIDFromRefObjectID(request.GetBody().GetObject().GetObjectID()) + if err != nil { + return nil, err + } + + var sTok *sessionSDK.Object + sTok, err = readSessionToken(cnr, obj, request.GetMetaHeader().GetSessionToken()) + if err != nil { + return nil, err + } + + bTok, err := originalBearerToken(request.GetMetaHeader()) + if err != nil { + return nil, err + } + + req := MetaWithToken{ + vheader: request.GetVerificationHeader(), + token: sTok, + bearer: bTok, + src: request, + } + + reqInfo, err := b.findRequestInfo(req, cnr, acl.OpObjectPut) + if err != nil { + return nil, err + } + + reqInfo.obj = obj + + if !b.checker.CheckBasicACL(reqInfo) || !b.checker.StickyBitCheck(reqInfo, idOwner) { + return nil, basicACLErr(reqInfo) + } + + if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return nil, eACLErr(reqInfo, err) + } + + return b.next.PutSingle(ctx, request) +} + func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { body := request.GetBody() if body == nil { @@ -481,7 +535,7 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe } var sTok *sessionSDK.Object - sTok, err = p.readSessionToken(cnr, obj, request) + sTok, err = readSessionToken(cnr, obj, request.GetMetaHeader().GetSessionToken()) if err != nil { return err } @@ -515,10 +569,10 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe return p.next.Send(ctx, request) } -func (p putStreamBasicChecker) readSessionToken(cnr cid.ID, obj *oid.ID, request *objectV2.PutRequest) (*sessionSDK.Object, error) { +func readSessionToken(cnr cid.ID, obj *oid.ID, tokV2 *session.Token) (*sessionSDK.Object, error) { var sTok *sessionSDK.Object - if tokV2 := request.GetMetaHeader().GetSessionToken(); tokV2 != nil { + if tokV2 != nil { sTok = new(sessionSDK.Object) err := sTok.ReadFromV2(*tokV2) diff --git a/pkg/services/object/acl/v2/util.go b/pkg/services/object/acl/v2/util.go index aa5d67584..feda6a3cf 100644 --- a/pkg/services/object/acl/v2/util.go +++ b/pkg/services/object/acl/v2/util.go @@ -44,6 +44,8 @@ func getContainerIDFromRequest(req any) (cid.ID, error) { idV2 = v.GetBody().GetAddress().GetContainerID() case *objectV2.GetRangeHashRequest: idV2 = v.GetBody().GetAddress().GetContainerID() + case *objectV2.PutSingleRequest: + idV2 = v.GetBody().GetObject().GetHeader().GetContainerID() default: return cid.ID{}, errors.New("unknown request type") } @@ -97,6 +99,10 @@ func originalSessionToken(header *sessionV2.RequestMetaHeader) (*sessionSDK.Obje // object reference's holders. Returns an error if object ID is missing in the request. func getObjectIDFromRequestBody(body interface{ GetAddress() *refsV2.Address }) (*oid.ID, error) { idV2 := body.GetAddress().GetObjectID() + return getObjectIDFromRefObjectID(idV2) +} + +func getObjectIDFromRefObjectID(idV2 *refsV2.ObjectID) (*oid.ID, error) { if idV2 == nil { return nil, errors.New("missing object ID") } @@ -118,7 +124,7 @@ func ownerFromToken(token *sessionSDK.Object) (*user.ID, *keys.PublicKey, error) } // 2. Then check if session token owner issued the session token - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var tokV2 sessionV2.Token token.WriteToV2(&tokV2) diff --git a/pkg/services/object/common.go b/pkg/services/object/common.go index 5b139d8eb..0d39dce0b 100644 --- a/pkg/services/object/common.go +++ b/pkg/services/object/common.go @@ -89,3 +89,11 @@ func (x *Common) GetRangeHash(ctx context.Context, req *objectV2.GetRangeHashReq return x.nextHandler.GetRangeHash(ctx, req) } + +func (x *Common) PutSingle(ctx context.Context, req *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { + if x.state.IsMaintenance() { + return nil, errMaintenance + } + + return x.nextHandler.PutSingle(ctx, req) +} diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index 91bc6b3d7..b10f045ee 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -8,8 +8,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + apiclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -28,11 +29,11 @@ type execCtx struct { log *logger.Logger - tombstone *object.Tombstone + tombstone *objectSDK.Tombstone - splitInfo *object.SplitInfo + splitInfo *objectSDK.SplitInfo - tombstoneObj *object.Object + tombstoneObj *objectSDK.Object } const ( @@ -75,8 +76,9 @@ func (exec *execCtx) newAddress(id oid.ID) oid.Address { } func (exec *execCtx) formSplitInfo(ctx context.Context) bool { - var err error + success := false + var err error exec.splitInfo, err = exec.svc.header.splitInfo(ctx, exec) switch { @@ -87,12 +89,17 @@ func (exec *execCtx) formSplitInfo(ctx context.Context) bool { exec.log.Debug(logs.DeleteCouldNotComposeSplitInfo, zap.String("error", err.Error()), ) - case err == nil: + case err == nil, apiclient.IsErrObjectAlreadyRemoved(err): + // IsErrObjectAlreadyRemoved check is required because splitInfo + // implicitly performs Head request that may return ObjectAlreadyRemoved + // status that is not specified for Delete + exec.status = statusOK exec.err = nil + success = true } - return err == nil + return success } func (exec *execCtx) collectMembers(ctx context.Context) (ok bool) { @@ -234,9 +241,9 @@ func (exec *execCtx) initTombstoneObject() bool { return false } - exec.tombstoneObj = object.New() + exec.tombstoneObj = objectSDK.New() exec.tombstoneObj.SetContainerID(exec.containerID()) - exec.tombstoneObj.SetType(object.TypeTombstone) + exec.tombstoneObj.SetType(objectSDK.TypeTombstone) exec.tombstoneObj.SetPayload(payload) tokenSession := exec.commonParameters().SessionToken() @@ -249,7 +256,7 @@ func (exec *execCtx) initTombstoneObject() bool { exec.tombstoneObj.SetOwnerID(&localUser) } - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(strconv.FormatUint(exec.tombstone.ExpirationEpoch(), 10)) diff --git a/pkg/services/object/delete/local.go b/pkg/services/object/delete/local.go index 34839b194..ad3e10bc6 100644 --- a/pkg/services/object/delete/local.go +++ b/pkg/services/object/delete/local.go @@ -4,7 +4,7 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -35,7 +35,7 @@ func (exec *execCtx) formTombstone(ctx context.Context) (ok bool) { return false } - exec.tombstone = object.NewTombstone() + exec.tombstone = objectSDK.NewTombstone() exec.tombstone.SetExpirationEpoch( exec.svc.netInfo.CurrentEpoch() + tsLifetime, ) diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index 11ff13b45..b74a4c7ba 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -9,7 +9,7 @@ import ( searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "go.uber.org/zap" @@ -41,7 +41,7 @@ type cfg struct { header interface { // must return (nil, nil) for PHY objects - splitInfo(context.Context, *execCtx) (*object.SplitInfo, error) + splitInfo(context.Context, *execCtx) (*objectSDK.SplitInfo, error) children(context.Context, *execCtx) ([]oid.ID, error) @@ -62,16 +62,22 @@ type cfg struct { keyStorage *util.KeyStorage } -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - } -} - // New creates, initializes and returns utility serving // Object.Get service requests. -func New(opts ...Option) *Service { - c := defaultCfg() +func New(gs *getsvc.Service, + ss *searchsvc.Service, + ps *putsvc.Service, + ni NetworkInfo, + ks *util.KeyStorage, + opts ...Option) *Service { + c := &cfg{ + log: &logger.Logger{Logger: zap.L()}, + header: &headSvcWrapper{s: gs}, + searcher: &searchSvcWrapper{s: ss}, + placer: &putSvcWrapper{s: ps}, + netInfo: ni, + keyStorage: ks, + } for i := range opts { opts[i](c) @@ -85,42 +91,6 @@ func New(opts ...Option) *Service { // WithLogger returns option to specify Delete service's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Delete service"))} - } -} - -// WithHeadService returns option to set Head service -// to work with object headers. -func WithHeadService(h *getsvc.Service) Option { - return func(c *cfg) { - c.header = (*headSvcWrapper)(h) - } -} - -// WithSearchService returns option to set search service. -func WithSearchService(s *searchsvc.Service) Option { - return func(c *cfg) { - c.searcher = (*searchSvcWrapper)(s) - } -} - -// WithPutService returns option to specify put service. -func WithPutService(p *putsvc.Service) Option { - return func(c *cfg) { - c.placer = (*putSvcWrapper)(p) - } -} - -// WithNetworkInfo returns option to set network information source. -func WithNetworkInfo(netInfo NetworkInfo) Option { - return func(c *cfg) { - c.netInfo = netInfo - } -} - -// WithKeyStorage returns option to set local private key storage. -func WithKeyStorage(ks *util.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = ks + c.log = &logger.Logger{Logger: l.With(zap.String("component", "objectSDK.Delete service"))} } } diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index f6341f02a..439abca2b 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -7,21 +7,27 @@ import ( getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -type headSvcWrapper getsvc.Service +type headSvcWrapper struct { + s *getsvc.Service +} -type searchSvcWrapper searchsvc.Service +type searchSvcWrapper struct { + s *searchsvc.Service +} -type putSvcWrapper putsvc.Service +type putSvcWrapper struct { + s *putsvc.Service +} type simpleIDWriter struct { ids []oid.ID } -func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oid.Address) (*object.Object, error) { +func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oid.Address) (*objectSDK.Object, error) { wr := getsvc.NewSimpleObjectWriter() p := getsvc.HeadPrm{} @@ -30,7 +36,7 @@ func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oi p.WithRawFlag(true) p.WithAddress(addr) - err := (*getsvc.Service)(w).Head(ctx, p) + err := w.s.Head(ctx, p) if err != nil { return nil, err } @@ -38,10 +44,10 @@ func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oi return wr.Object(), nil } -func (w *headSvcWrapper) splitInfo(ctx context.Context, exec *execCtx) (*object.SplitInfo, error) { +func (w *headSvcWrapper) splitInfo(ctx context.Context, exec *execCtx) (*objectSDK.SplitInfo, error) { _, err := w.headAddress(ctx, exec, exec.address()) - var errSplitInfo *object.SplitInfoError + var errSplitInfo *objectSDK.SplitInfoError switch { case err == nil: @@ -83,8 +89,8 @@ func (w *headSvcWrapper) previous(ctx context.Context, exec *execCtx, id oid.ID) } func (w *searchSvcWrapper) splitMembers(ctx context.Context, exec *execCtx) ([]oid.ID, error) { - fs := object.SearchFilters{} - fs.AddSplitIDFilter(object.MatchStringEqual, exec.splitInfo.SplitID()) + fs := objectSDK.SearchFilters{} + fs.AddSplitIDFilter(objectSDK.MatchStringEqual, exec.splitInfo.SplitID()) wr := new(simpleIDWriter) @@ -94,7 +100,7 @@ func (w *searchSvcWrapper) splitMembers(ctx context.Context, exec *execCtx) ([]o p.WithContainerID(exec.containerID()) p.WithSearchFilters(fs) - err := (*searchsvc.Service)(w).Search(ctx, p) + err := w.s.Search(ctx, p) if err != nil { return nil, err } @@ -109,7 +115,7 @@ func (s *simpleIDWriter) WriteIDs(ids []oid.ID) error { } func (w *putSvcWrapper) put(ctx context.Context, exec *execCtx) (*oid.ID, error) { - streamer, err := (*putsvc.Service)(w).Put() + streamer, err := w.s.Put() if err != nil { return nil, err } diff --git a/pkg/services/object/delete/v2/service.go b/pkg/services/object/delete/v2/service.go index 51759c5df..10dcd0e87 100644 --- a/pkg/services/object/delete/v2/service.go +++ b/pkg/services/object/delete/v2/service.go @@ -9,26 +9,13 @@ import ( // Service implements Delete operation of Object service v2. type Service struct { - *cfg -} - -// Option represents Service constructor option. -type Option func(*cfg) - -type cfg struct { svc *deletesvc.Service } // NewService constructs Service instance from provided options. -func NewService(opts ...Option) *Service { - c := new(cfg) - - for i := range opts { - opts[i](c) - } - +func NewService(svc *deletesvc.Service) *Service { return &Service{ - cfg: c, + svc: svc, } } @@ -51,9 +38,3 @@ func (s *Service) Delete(ctx context.Context, req *objectV2.DeleteRequest) (*obj return resp, nil } - -func WithInternalService(v *deletesvc.Service) Option { - return func(c *cfg) { - c.svc = v - } -} diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 457193a59..e3037a70b 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -5,7 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" ) @@ -74,7 +74,7 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { localStorage: s.localStorage, prm: prm, - infoSplit: object.NewSplitInfo(), + infoSplit: objectSDK.NewSplitInfo(), } exec.setLogger(s.log) diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 62dde3281..03ede58cc 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -4,8 +4,8 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 69bdbf271..e3464f941 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -4,9 +4,9 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index d11f94b26..40aa3f62e 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -8,7 +8,6 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -17,21 +16,22 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) type getRequestForwarder struct { - OnceResign *sync.Once - OnceHeaderSending *sync.Once + OnceResign sync.Once + OnceHeaderSending sync.Once GlobalProgress int Key *ecdsa.PrivateKey Request *objectV2.GetRequest Stream *streamObjectWriter } -func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*objectSDK.Object, error) { ctx, span := tracing.StartSpanFromContext(ctx, "getRequestForwarder.forwardRequestToNode", trace.WithAttributes(attribute.String("address", addr.String())), ) @@ -85,7 +85,7 @@ func (f *getRequestForwarder) writeHeader(ctx context.Context, v *objectV2.GetOb var err error f.OnceHeaderSending.Do(func() { - err = f.Stream.WriteHeader(ctx, object.NewFromV2(obj)) + err = f.Stream.WriteHeader(ctx, objectSDK.NewFromV2(obj)) }) if err != nil { return errCouldNotWriteObjHeader(err) @@ -164,8 +164,8 @@ func (f *getRequestForwarder) readStream(ctx context.Context, c client.MultiAddr localProgress += len(origChunk) f.GlobalProgress += len(chunk) case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - return object.NewSplitInfoError(si) + si := objectSDK.NewSplitInfoFromV2(v) + return objectSDK.NewSplitInfoError(si) } } return nil diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 6c744b23a..8a56c59a6 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -8,7 +8,6 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -17,20 +16,21 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) type getRangeRequestForwarder struct { - OnceResign *sync.Once + OnceResign sync.Once GlobalProgress int Key *ecdsa.PrivateKey Request *objectV2.GetRangeRequest Stream *streamObjectRangeWriter } -func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*objectSDK.Object, error) { ctx, span := tracing.StartSpanFromContext(ctx, "getRangeRequestForwarder.forwardRequestToNode", trace.WithAttributes(attribute.String("address", addr.String())), ) @@ -130,8 +130,8 @@ func (f *getRangeRequestForwarder) readStream(ctx context.Context, rangeStream * localProgress += len(origChunk) f.GlobalProgress += len(chunk) case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - return object.NewSplitInfoError(si) + si := objectSDK.NewSplitInfoFromV2(v) + return objectSDK.NewSplitInfoError(si) } } return nil diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index 0c91ec5d8..a1bce1517 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -6,7 +6,6 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -15,8 +14,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -25,12 +25,12 @@ import ( type headRequestForwarder struct { Request *objectV2.HeadRequest Response *objectV2.HeadResponse - OnceResign *sync.Once + OnceResign sync.Once ObjectAddr oid.Address Key *ecdsa.PrivateKey } -func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*objectSDK.Object, error) { ctx, span := tracing.StartSpanFromContext(ctx, "headRequestForwarder.forwardRequestToNode", trace.WithAttributes(attribute.String("address", addr.String())), ) @@ -82,15 +82,15 @@ func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr ne return nil, err } case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - return nil, object.NewSplitInfoError(si) + si := objectSDK.NewSplitInfoFromV2(v) + return nil, objectSDK.NewSplitInfoError(si) } objv2 := new(objectV2.Object) objv2.SetHeader(hdr) objv2.SetSignature(idSig) - obj := object.NewFromV2(objv2) + obj := objectSDK.NewFromV2(objv2) obj.SetID(f.ObjectAddr.Object()) return obj, nil diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index 1bd8befaf..d4bce178a 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -8,7 +8,7 @@ import ( objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) // Service implements Get operation of Object service v2. @@ -47,7 +47,7 @@ func (s *Service) Get(req *objectV2.GetRequest, stream objectSvc.GetObjectStream err = s.svc.Get(stream.Context(), *p) - var splitErr *object.SplitInfoError + var splitErr *objectSDK.SplitInfoError switch { case errors.As(err, &splitErr): @@ -66,7 +66,7 @@ func (s *Service) GetRange(req *objectV2.GetRangeRequest, stream objectSvc.GetOb err = s.svc.GetRange(stream.Context(), *p) - var splitErr *object.SplitInfoError + var splitErr *objectSDK.SplitInfoError switch { case errors.As(err, &splitErr): @@ -103,7 +103,7 @@ func (s *Service) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV err = s.svc.Head(ctx, *p) - var splitErr *object.SplitInfoError + var splitErr *objectSDK.SplitInfoError if errors.As(err, &splitErr) { setSplitInfoHeadResponse(splitErr.SplitInfo(), resp) diff --git a/pkg/services/object/get/v2/streamer.go b/pkg/services/object/get/v2/streamer.go index 4347ef416..ce9a5c767 100644 --- a/pkg/services/object/get/v2/streamer.go +++ b/pkg/services/object/get/v2/streamer.go @@ -5,7 +5,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) type streamObjectWriter struct { @@ -16,7 +16,7 @@ type streamObjectRangeWriter struct { objectSvc.GetObjectRangeStream } -func (s *streamObjectWriter) WriteHeader(_ context.Context, obj *object.Object) error { +func (s *streamObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Object) error { p := new(objectV2.GetObjectPartInit) objV2 := obj.ToV2() diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 91e7a96a2..9ecc9167f 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -5,7 +5,6 @@ import ( "crypto/sha256" "errors" "hash" - "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" @@ -17,7 +16,7 @@ import ( getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" versionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/tzhash/tz" @@ -59,12 +58,10 @@ func (s *Service) toPrm(req *objectV2.GetRequest, stream objectSvc.GetObjectStre } forwarder := &getRequestForwarder{ - OnceResign: &sync.Once{}, - OnceHeaderSending: &sync.Once{}, - GlobalProgress: 0, - Key: key, - Request: req, - Stream: streamWrapper, + GlobalProgress: 0, + Key: key, + Request: req, + Stream: streamWrapper, } p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequestToNode)) @@ -101,7 +98,7 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get p.WithAddress(addr) p.WithRawFlag(body.GetRaw()) p.SetChunkWriter(streamWrapper) - p.SetRange(object.NewRangeFromV2(body.GetRange())) + p.SetRange(objectSDK.NewRangeFromV2(body.GetRange())) err = p.Validate() if err != nil { @@ -115,7 +112,6 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get } forwarder := &getRangeRequestForwarder{ - OnceResign: &sync.Once{}, GlobalProgress: 0, Key: key, Request: req, @@ -171,10 +167,10 @@ func (s *Service) toHashRangePrm(req *objectV2.GetRangeHashRequest) (*getsvc.Ran } rngsV2 := body.GetRanges() - rngs := make([]object.Range, len(rngsV2)) + rngs := make([]objectSDK.Range, len(rngsV2)) for i := range rngsV2 { - rngs[i] = *object.NewRangeFromV2(&rngsV2[i]) + rngs[i] = *objectSDK.NewRangeFromV2(&rngsV2[i]) } p.SetRangeList(rngs) @@ -202,7 +198,7 @@ type headResponseWriter struct { body *objectV2.HeadResponseBody } -func (w *headResponseWriter) WriteHeader(_ context.Context, hdr *object.Object) error { +func (w *headResponseWriter) WriteHeader(_ context.Context, hdr *objectSDK.Object) error { if w.mainOnly { w.body.SetHeaderPart(toShortObjectHeader(hdr)) } else { @@ -254,7 +250,6 @@ func (s *Service) toHeadPrm(req *objectV2.HeadRequest, resp *objectV2.HeadRespon forwarder := &headRequestForwarder{ Request: req, Response: resp, - OnceResign: &sync.Once{}, ObjectAddr: objAddr, Key: key, } @@ -264,7 +259,7 @@ func (s *Service) toHeadPrm(req *objectV2.HeadRequest, resp *objectV2.HeadRespon return p, nil } -func splitInfoResponse(info *object.SplitInfo) *objectV2.GetResponse { +func splitInfoResponse(info *objectSDK.SplitInfo) *objectV2.GetResponse { resp := new(objectV2.GetResponse) body := new(objectV2.GetResponseBody) @@ -275,7 +270,7 @@ func splitInfoResponse(info *object.SplitInfo) *objectV2.GetResponse { return resp } -func splitInfoRangeResponse(info *object.SplitInfo) *objectV2.GetRangeResponse { +func splitInfoRangeResponse(info *objectSDK.SplitInfo) *objectV2.GetRangeResponse { resp := new(objectV2.GetRangeResponse) body := new(objectV2.GetRangeResponseBody) @@ -286,7 +281,7 @@ func splitInfoRangeResponse(info *object.SplitInfo) *objectV2.GetRangeResponse { return resp } -func setSplitInfoHeadResponse(info *object.SplitInfo, resp *objectV2.HeadResponse) { +func setSplitInfoHeadResponse(info *objectSDK.SplitInfo, resp *objectV2.HeadResponse) { resp.GetBody().SetHeaderPart(info.ToV2()) } @@ -302,7 +297,7 @@ func toHashResponse(typ refs.ChecksumType, res *getsvc.RangeHashRes) *objectV2.G return resp } -func toFullObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { +func toFullObjectHeader(hdr *objectSDK.Object) objectV2.GetHeaderPart { obj := hdr.ToV2() hs := new(objectV2.HeaderWithSignature) @@ -312,7 +307,7 @@ func toFullObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { return hs } -func toShortObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { +func toShortObjectHeader(hdr *objectSDK.Object) objectV2.GetHeaderPart { hdrV2 := hdr.ToV2().GetHeader() sh := new(objectV2.ShortHeader) @@ -327,11 +322,11 @@ func toShortObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { return sh } -func groupAddressRequestForwarder(f func(context.Context, network.Address, client.MultiAddressClient, []byte) (*object.Object, error)) getsvc.RequestForwarder { - return func(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) (*object.Object, error) { +func groupAddressRequestForwarder(f func(context.Context, network.Address, client.MultiAddressClient, []byte) (*objectSDK.Object, error)) getsvc.RequestForwarder { + return func(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) (*objectSDK.Object, error) { var ( firstErr error - res *object.Object + res *objectSDK.Object key = info.PublicKey() ) diff --git a/pkg/services/object/get/writer.go b/pkg/services/object/get/writer.go index 78af5db41..1b842adeb 100644 --- a/pkg/services/object/get/writer.go +++ b/pkg/services/object/get/writer.go @@ -4,7 +4,7 @@ import ( "context" "io" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) // ChunkWriter is an interface of target component @@ -16,7 +16,7 @@ type ChunkWriter interface { // HeaderWriter is an interface of target component // to write object header. type HeaderWriter interface { - WriteHeader(context.Context, *object.Object) error + WriteHeader(context.Context, *objectSDK.Object) error } // ObjectWriter is an interface of target component to write object. @@ -26,7 +26,7 @@ type ObjectWriter interface { } type SimpleObjectWriter struct { - obj *object.Object + obj *objectSDK.Object pld []byte } @@ -45,11 +45,11 @@ type hasherWrapper struct { func NewSimpleObjectWriter() *SimpleObjectWriter { return &SimpleObjectWriter{ - obj: object.New(), + obj: objectSDK.New(), } } -func (s *SimpleObjectWriter) WriteHeader(_ context.Context, obj *object.Object) error { +func (s *SimpleObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Object) error { s.obj = obj s.pld = make([]byte, 0, obj.PayloadSize()) @@ -62,7 +62,7 @@ func (s *SimpleObjectWriter) WriteChunk(_ context.Context, p []byte) error { return nil } -func (s *SimpleObjectWriter) Object() *object.Object { +func (s *SimpleObjectWriter) Object() *objectSDK.Object { if len(s.pld) > 0 { s.obj.SetPayload(s.pld) } @@ -74,7 +74,7 @@ func (w *partWriter) WriteChunk(ctx context.Context, p []byte) error { return w.chunkWriter.WriteChunk(ctx, p) } -func (w *partWriter) WriteHeader(ctx context.Context, o *object.Object) error { +func (w *partWriter) WriteHeader(ctx context.Context, o *objectSDK.Object) error { return w.headWriter.WriteHeader(ctx, o) } diff --git a/pkg/services/object/head/remote.go b/pkg/services/object/head/remote.go index bcba181f2..c9c17d4d8 100644 --- a/pkg/services/object/head/remote.go +++ b/pkg/services/object/head/remote.go @@ -10,7 +10,7 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -64,7 +64,7 @@ func (p *RemoteHeadPrm) WithObjectAddress(v oid.Address) *RemoteHeadPrm { } // Head requests object header from the remote node. -func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*object.Object, error) { +func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*objectSDK.Object, error) { key, err := h.keyStorage.GetKey(nil) if err != nil { return nil, fmt.Errorf("(%T) could not receive private key: %w", h, err) diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 6beb67476..73f4ff7c4 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -8,13 +8,13 @@ import ( "fmt" "io" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" ) @@ -77,11 +77,11 @@ type readPrmCommon struct { commonPrm } -// SetNetmapEpoch sets the epoch number to be used to locate the object. +// SetNetmapEpoch sets the epoch number to be used to locate the objectSDK. // // By default current epoch on the server will be used. func (x *readPrmCommon) SetNetmapEpoch(_ uint64) { - // FIXME: (neofs-node#1194) not supported by client + // FIXME(@fyrchik): https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/465 } // GetObjectPrm groups parameters of GetObject operation. @@ -111,11 +111,11 @@ func (x *GetObjectPrm) SetAddress(addr oid.Address) { // GetObjectRes groups the resulting values of GetObject operation. type GetObjectRes struct { - obj *object.Object + obj *objectSDK.Object } -// Object returns requested object. -func (x GetObjectRes) Object() *object.Object { +// Object returns requested objectSDK. +func (x GetObjectRes) Object() *objectSDK.Object { return x.obj } @@ -125,10 +125,10 @@ func (x GetObjectRes) Object() *object.Object { // // Returns any error which prevented the operation from completing correctly in error return. // Returns: -// - error of type *object.SplitInfoError if object raw flag is set and requested object is virtual; +// - error of type *objectSDK.SplitInfoError if object raw flag is set and requested object is virtual; // - error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed. // -// GetObject ignores the provided session if it is not related to the requested object. +// GetObject ignores the provided session if it is not related to the requested objectSDK. func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { // here we ignore session if it is opened for other object since such // request will almost definitely fail. The case can occur, for example, @@ -155,7 +155,7 @@ func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { return nil, fmt.Errorf("init object reading: %w", err) } - var obj object.Object + var obj objectSDK.Object if !rdr.ReadHeader(&obj) { res, err := rdr.Close() @@ -210,11 +210,11 @@ func (x *HeadObjectPrm) SetAddress(addr oid.Address) { // HeadObjectRes groups the resulting values of GetObject operation. type HeadObjectRes struct { - hdr *object.Object + hdr *objectSDK.Object } // Header returns requested object header. -func (x HeadObjectRes) Header() *object.Object { +func (x HeadObjectRes) Header() *objectSDK.Object { return x.hdr } @@ -225,10 +225,10 @@ func (x HeadObjectRes) Header() *object.Object { // Returns any error which prevented the operation from completing correctly in error return. // Returns: // -// error of type *object.SplitInfoError if object raw flag is set and requested object is virtual; +// error of type *objectSDK.SplitInfoError if object raw flag is set and requested object is virtual; // error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed. // -// HeadObject ignores the provided session if it is not related to the requested object. +// HeadObject ignores the provided session if it is not related to the requested objectSDK. func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) { if prm.local { prm.cliPrm.MarkLocal() @@ -255,7 +255,7 @@ func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) return nil, fmt.Errorf("read object header from FrostFS: %w", err) } - var hdr object.Object + var hdr objectSDK.Object if !cliRes.ReadHeader(&hdr) { return nil, errors.New("missing object header in the response") @@ -296,7 +296,7 @@ func (x *PayloadRangePrm) SetAddress(addr oid.Address) { // SetRange range of the object payload to be read. // // Required parameter. -func (x *PayloadRangePrm) SetRange(rng *object.Range) { +func (x *PayloadRangePrm) SetRange(rng *objectSDK.Range) { x.cliPrm.SetOffset(rng.GetOffset()) x.ln = rng.GetLength() } @@ -323,11 +323,11 @@ const maxInitialBufferSize = 1024 * 1024 // 1 MiB // Returns any error which prevented the operation from completing correctly in error return. // Returns: // -// error of type *object.SplitInfoError if object raw flag is set and requested object is virtual; +// error of type *objectSDK.SplitInfoError if object raw flag is set and requested object is virtual; // error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed; // error of type *apistatus.ObjectOutOfRange if the requested range is too big. // -// PayloadRange ignores the provided session if it is not related to the requested object. +// PayloadRange ignores the provided session if it is not related to the requested objectSDK. func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, error) { if prm.local { prm.cliPrm.MarkLocal() @@ -377,13 +377,13 @@ func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, e type PutObjectPrm struct { commonPrm - obj *object.Object + obj *objectSDK.Object } // SetObject sets object to be stored. // // Required parameter. -func (x *PutObjectPrm) SetObject(obj *object.Object) { +func (x *PutObjectPrm) SetObject(obj *objectSDK.Object) { x.obj = obj } @@ -392,7 +392,7 @@ type PutObjectRes struct { id oid.ID } -// ID returns identifier of the stored object. +// ID returns identifier of the stored objectSDK. func (x PutObjectRes) ID() oid.ID { return x.id } @@ -429,11 +429,11 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { return nil, fmt.Errorf("init object writing on client: %w", err) } - if w.WriteHeader(*prm.obj) { - w.WritePayloadChunk(prm.obj.Payload()) + if w.WriteHeader(ctx, *prm.obj) { + w.WritePayloadChunk(ctx, prm.obj.Payload()) } - cliRes, err := w.Close() + cliRes, err := w.Close(ctx) if err == nil { err = apistatus.ErrFromStatus(cliRes.Status()) } else { @@ -449,6 +449,54 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { }, nil } +// PutObjectSingle saves the object in local storage of the remote node with PutSingle RPC. +// +// Client and key must be set. +// +// Returns any error which prevented the operation from completing correctly in error return. +func PutObjectSingle(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "client.PutObjectSingle") + defer span.End() + + objID, isSet := prm.obj.ID() + if !isSet { + return nil, errors.New("missing object id") + } + + var prmCli client.PrmObjectPutSingle + + prmCli.ExecuteLocal() + + if prm.key != nil { + prmCli.UseKey(prm.key) + } + + if prm.tokenSession != nil { + prmCli.WithinSession(*prm.tokenSession) + } + + if prm.tokenBearer != nil { + prmCli.WithBearerToken(*prm.tokenBearer) + } + + prmCli.WithXHeaders(prm.xHeaders...) + prmCli.SetObject(prm.obj.ToV2()) + + res, err := prm.cli.ObjectPutSingle(ctx, prmCli) + if err != nil { + ReportError(prm.cli, err) + return nil, fmt.Errorf("put single object on client: %w", err) + } + + if err = apistatus.ErrFromStatus(res.Status()); err != nil { + return nil, fmt.Errorf("put single object via client: %w", err) + } + + return &PutObjectRes{ + id: objID, + }, nil +} + // SearchObjectsPrm groups parameters of SearchObjects operation. type SearchObjectsPrm struct { readPrmCommon @@ -464,7 +512,7 @@ func (x *SearchObjectsPrm) SetContainerID(id cid.ID) { } // SetFilters sets search filters. -func (x *SearchObjectsPrm) SetFilters(fs object.SearchFilters) { +func (x *SearchObjectsPrm) SetFilters(fs objectSDK.SearchFilters) { x.cliPrm.SetFilters(fs) } diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index 3ea16dafd..f972f43ae 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -28,24 +28,8 @@ type ( } MetricRegister interface { - IncGetReqCounter(success bool) - IncPutReqCounter(success bool) - IncHeadReqCounter(success bool) - IncSearchReqCounter(success bool) - IncDeleteReqCounter(success bool) - IncRangeReqCounter(success bool) - IncRangeHashReqCounter(success bool) - - AddGetReqDuration(time.Duration) - AddPutReqDuration(time.Duration) - AddHeadReqDuration(time.Duration) - AddSearchReqDuration(time.Duration) - AddDeleteReqDuration(time.Duration) - AddRangeReqDuration(time.Duration) - AddRangeHashReqDuration(time.Duration) - - AddPutPayload(int) - AddGetPayload(int) + AddRequestDuration(string, time.Duration, bool) + AddPayloadSize(string, int) } ) @@ -61,8 +45,7 @@ func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (er if m.enabled { t := time.Now() defer func() { - m.metrics.IncGetReqCounter(err == nil) - m.metrics.AddGetReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Get", time.Since(t), err == nil) }() err = m.next.Get(req, &getStreamMetric{ ServerStream: stream, @@ -93,14 +76,29 @@ func (m MetricCollector) Put() (PutObjectStream, error) { return m.next.Put() } +func (m MetricCollector) PutSingle(ctx context.Context, request *object.PutSingleRequest) (*object.PutSingleResponse, error) { + if m.enabled { + t := time.Now() + + res, err := m.next.PutSingle(ctx, request) + + m.metrics.AddRequestDuration("PutSingle", time.Since(t), err == nil) + if err == nil { + m.metrics.AddPayloadSize("PutSingle", len(request.GetBody().GetObject().GetPayload())) + } + + return res, err + } + return m.next.PutSingle(ctx, request) +} + func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { if m.enabled { t := time.Now() res, err := m.next.Head(ctx, request) - m.metrics.IncHeadReqCounter(err == nil) - m.metrics.AddHeadReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Head", time.Since(t), err == nil) return res, err } @@ -113,8 +111,7 @@ func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream) err := m.next.Search(req, stream) - m.metrics.IncSearchReqCounter(err == nil) - m.metrics.AddSearchReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Search", time.Since(t), err == nil) return err } @@ -127,8 +124,7 @@ func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteReque res, err := m.next.Delete(ctx, request) - m.metrics.IncDeleteReqCounter(err == nil) - m.metrics.AddDeleteReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Delete", time.Since(t), err == nil) return res, err } return m.next.Delete(ctx, request) @@ -140,8 +136,7 @@ func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectR err := m.next.GetRange(req, stream) - m.metrics.IncRangeReqCounter(err == nil) - m.metrics.AddRangeReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("GetRange", time.Since(t), err == nil) return err } @@ -154,8 +149,7 @@ func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRa res, err := m.next.GetRangeHash(ctx, request) - m.metrics.IncRangeHashReqCounter(err == nil) - m.metrics.AddRangeHashReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("GetRangeHash", time.Since(t), err == nil) return res, err } @@ -173,7 +167,7 @@ func (m *MetricCollector) Disable() { func (s getStreamMetric) Send(resp *object.GetResponse) error { chunk, ok := resp.GetBody().GetObjectPart().(*object.GetObjectPartChunk) if ok { - s.metrics.AddGetPayload(len(chunk.GetChunk())) + s.metrics.AddPayloadSize("Get", len(chunk.GetChunk())) } return s.stream.Send(resp) @@ -182,7 +176,7 @@ func (s getStreamMetric) Send(resp *object.GetResponse) error { func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error { chunk, ok := req.GetBody().GetObjectPart().(*object.PutObjectPartChunk) if ok { - s.metrics.AddPutPayload(len(chunk.GetChunk())) + s.metrics.AddPayloadSize("Put", len(chunk.GetChunk())) } return s.stream.Send(ctx, req) @@ -191,8 +185,7 @@ func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { res, err := s.stream.CloseAndRecv(ctx) - s.metrics.IncPutReqCounter(err == nil) - s.metrics.AddPutReqDuration(time.Since(s.start)) + s.metrics.AddRequestDuration("Put", time.Since(s.start), err == nil) return res, err } diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index b24218621..cf5cc558e 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -18,15 +18,12 @@ import ( ) type preparedObjectTarget interface { - WriteObject(*objectSDK.Object, object.ContentMeta) error - Close(ctx context.Context) (*transformer.AccessIdentifiers, error) + WriteObject(context.Context, *objectSDK.Object, object.ContentMeta) error } type distributedTarget struct { traversal traversal - remotePool, localPool util.WorkerPool - obj *objectSDK.Object objMeta object.ContentMeta @@ -34,7 +31,7 @@ type distributedTarget struct { nodeTargetInitializer func(nodeDesc) preparedObjectTarget - isLocalKey func([]byte) bool + getWorkerPool func([]byte) (util.WorkerPool, bool) relay func(context.Context, nodeDesc) error @@ -137,10 +134,24 @@ func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdent t.obj.SetPayload(t.payload.Data) + if err := t.WriteObject(ctx, t.obj); err != nil { + return nil, err + } + + id, _ := t.obj.ID() + return &transformer.AccessIdentifiers{ + SelfID: id, + }, nil +} + +// WriteObject implements the transformer.ObjectWriter interface. +func (t *distributedTarget) WriteObject(ctx context.Context, obj *objectSDK.Object) error { + t.obj = obj + var err error if t.objMeta, err = t.fmt.ValidateContent(t.obj); err != nil { - return nil, fmt.Errorf("(%T) could not validate payload content: %w", t, err) + return fmt.Errorf("(%T) could not validate payload content: %w", t, err) } if len(t.obj.Children()) > 0 { @@ -158,22 +169,21 @@ func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error target := t.nodeTargetInitializer(node) - if err := target.WriteObject(t.obj, t.objMeta); err != nil { + err := target.WriteObject(ctx, t.obj, t.objMeta) + if err != nil { return fmt.Errorf("could not write header: %w", err) - } else if _, err := target.Close(ctx); err != nil { - return fmt.Errorf("could not close object stream: %w", err) } return nil } -func (t *distributedTarget) iteratePlacement(ctx context.Context) (*transformer.AccessIdentifiers, error) { +func (t *distributedTarget) iteratePlacement(ctx context.Context) error { id, _ := t.obj.ID() traverser, err := placement.NewTraverser( append(t.traversal.opts, placement.ForObject(id))..., ) if err != nil { - return nil, fmt.Errorf("(%T) could not create object placement traverser: %w", t, err) + return fmt.Errorf("(%T) could not create object placement traverser: %w", t, err) } resErr := &atomic.Value{} @@ -192,23 +202,19 @@ func (t *distributedTarget) iteratePlacement(ctx context.Context) (*transformer. if !traverser.Success() { var err errIncompletePut err.singleErr, _ = resErr.Load().(error) - return nil, err + return err } // perform additional container broadcast if needed if t.traversal.submitPrimaryPlacementFinish() { - _, err = t.iteratePlacement(ctx) + err = t.iteratePlacement(ctx) if err != nil { t.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) // we don't fail primary operation because of broadcast failure } } - id, _ = t.obj.ID() - - return &transformer.AccessIdentifiers{ - SelfID: id, - }, nil + return nil } func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, resErr *atomic.Value) bool { @@ -223,13 +229,8 @@ func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *pla wg.Add(1) addr := addrs[i] - isLocal := t.isLocalKey(addr.PublicKey()) - - workerPool := t.remotePool - if isLocal { - workerPool = t.localPool - } + workerPool, isLocal := t.getWorkerPool(addr.PublicKey()) if err := workerPool.Submit(func() { defer wg.Done() diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index f07122729..54649adc7 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -5,16 +5,15 @@ import ( "fmt" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) // ObjectStorage is an object storage interface. type ObjectStorage interface { // Put must save passed object // and return any appeared error. - Put(context.Context, *object.Object) error + Put(context.Context, *objectSDK.Object) error // Delete must delete passed objects // and return any appeared error. Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error @@ -27,41 +26,26 @@ type ObjectStorage interface { type localTarget struct { storage ObjectStorage - - obj *object.Object - meta objectCore.ContentMeta } -func (t *localTarget) WriteObject(obj *object.Object, meta objectCore.ContentMeta) error { - t.obj = obj - t.meta = meta - - return nil -} - -func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { - switch t.meta.Type() { - case object.TypeTombstone: - err := t.storage.Delete(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) +func (t localTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, meta objectCore.ContentMeta) error { + switch meta.Type() { + case objectSDK.TypeTombstone: + err := t.storage.Delete(ctx, objectCore.AddressOf(obj), meta.Objects()) if err != nil { - return nil, fmt.Errorf("could not delete objects from tombstone locally: %w", err) + return fmt.Errorf("could not delete objects from tombstone locally: %w", err) } - case object.TypeLock: - err := t.storage.Lock(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) + case objectSDK.TypeLock: + err := t.storage.Lock(ctx, objectCore.AddressOf(obj), meta.Objects()) if err != nil { - return nil, fmt.Errorf("could not lock object from lock objects locally: %w", err) + return fmt.Errorf("could not lock object from lock objects locally: %w", err) } default: // objects that do not change meta storage } - if err := t.storage.Put(ctx, t.obj); err != nil { //TODO - return nil, fmt.Errorf("(%T) could not put object to local storage: %w", t, err) + if err := t.storage.Put(ctx, obj); err != nil { + return fmt.Errorf("(%T) could not put object to local storage: %w", t, err) } - - id, _ := t.obj.ID() - - return &transformer.AccessIdentifiers{ - SelfID: id, - }, nil + return nil } diff --git a/pkg/services/object/put/pool.go b/pkg/services/object/put/pool.go index 5726856e5..ebe214caf 100644 --- a/pkg/services/object/put/pool.go +++ b/pkg/services/object/put/pool.go @@ -4,7 +4,10 @@ import ( "sync" ) -const defaultAllocSize = 1024 +const ( + defaultAllocSize = 1024 + poolSliceMaxSize = 128 * 1024 +) type payload struct { Data []byte @@ -19,6 +22,9 @@ func getPayload() *payload { } func putPayload(p *payload) { + if cap(p.Data) > poolSliceMaxSize { + return + } p.Data = p.Data[:0] putBytesPool.Put(p) } diff --git a/pkg/services/object/put/prm.go b/pkg/services/object/put/prm.go index c8d1b29a2..52a7c102c 100644 --- a/pkg/services/object/put/prm.go +++ b/pkg/services/object/put/prm.go @@ -7,13 +7,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) type PutInitPrm struct { common *util.CommonPrm - hdr *object.Object + hdr *objectSDK.Object cnr containerSDK.Container @@ -34,7 +34,7 @@ func (p *PutInitPrm) WithCommonPrm(v *util.CommonPrm) *PutInitPrm { return p } -func (p *PutInitPrm) WithObject(v *object.Object) *PutInitPrm { +func (p *PutInitPrm) WithObject(v *objectSDK.Object) *PutInitPrm { if p != nil { p.hdr = v } diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index bcc566b74..ee8d64e7a 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -11,8 +11,9 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type remoteTarget struct { @@ -22,8 +23,6 @@ type remoteTarget struct { nodeInfo clientcore.NodeInfo - obj *object.Object - clientConstructor ClientConstructor } @@ -39,19 +38,13 @@ type RemoteSender struct { type RemotePutPrm struct { node netmap.NodeInfo - obj *object.Object + obj *objectSDK.Object } -func (t *remoteTarget) WriteObject(obj *object.Object, _ objectcore.ContentMeta) error { - t.obj = obj - - return nil -} - -func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { +func (t *remoteTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, _ objectcore.ContentMeta) error { c, err := t.clientConstructor.Get(t.nodeInfo) if err != nil { - return nil, fmt.Errorf("(%T) could not create SDK client %s: %w", t, t.nodeInfo, err) + return fmt.Errorf("(%T) could not create SDK client %s: %w", t, t.nodeInfo, err) } var prm internalclient.PutObjectPrm @@ -61,14 +54,30 @@ func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifier prm.SetSessionToken(t.commonPrm.SessionToken()) prm.SetBearerToken(t.commonPrm.BearerToken()) prm.SetXHeaders(t.commonPrm.XHeaders()) - prm.SetObject(t.obj) + prm.SetObject(obj) - res, err := internalclient.PutObject(ctx, prm) - if err != nil { - return nil, fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) + err = t.putSingle(ctx, prm) + if status.Code(err) != codes.Unimplemented { + return err } - return &transformer.AccessIdentifiers{SelfID: res.ID()}, nil + return t.putStream(ctx, prm) +} + +func (t *remoteTarget) putStream(ctx context.Context, prm internalclient.PutObjectPrm) error { + _, err := internalclient.PutObject(ctx, prm) + if err != nil { + return fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) + } + return nil +} + +func (t *remoteTarget) putSingle(ctx context.Context, prm internalclient.PutObjectPrm) error { + _, err := internalclient.PutObjectSingle(ctx, prm) + if err != nil { + return fmt.Errorf("(%T) could not put single object to %s: %w", t, t.nodeInfo.AddressGroup(), err) + } + return nil } // NewRemoteSender creates, initializes and returns new RemoteSender instance. @@ -89,7 +98,7 @@ func (p *RemotePutPrm) WithNodeInfo(v netmap.NodeInfo) *RemotePutPrm { } // WithObject sets transferred object. -func (p *RemotePutPrm) WithObject(v *object.Object) *RemotePutPrm { +func (p *RemotePutPrm) WithObject(v *objectSDK.Object) *RemotePutPrm { if p != nil { p.obj = v } @@ -114,9 +123,7 @@ func (s *RemoteSender) PutObject(ctx context.Context, p *RemotePutPrm) error { return fmt.Errorf("parse client node info: %w", err) } - if err := t.WriteObject(p.obj, objectcore.ContentMeta{}); err != nil { - return fmt.Errorf("(%T) could not send object header: %w", s, err) - } else if _, err := t.Close(ctx); err != nil { + if err := t.WriteObject(ctx, p.obj, objectcore.ContentMeta{}); err != nil { return fmt.Errorf("(%T) could not send object: %w", s, err) } diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 567a3fea1..7f2600f9c 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -46,8 +46,6 @@ type cfg struct { fmtValidator *object.FormatValidator - fmtValidatorOpts []object.FormatValidatorOption - networkState netmap.State clientConstructor ClientConstructor @@ -55,22 +53,34 @@ type cfg struct { log *logger.Logger } -func defaultCfg() *cfg { - return &cfg{ - remotePool: util.NewPseudoWorkerPool(), - localPool: util.NewPseudoWorkerPool(), - log: &logger.Logger{Logger: zap.L()}, +func NewService(ks *objutil.KeyStorage, + cc ClientConstructor, + ms MaxSizeSource, + os ObjectStorage, + cs container.Source, + ns netmap.Source, + nk netmap.AnnouncedKeys, + nst netmap.State, + opts ...Option) *Service { + c := &cfg{ + remotePool: util.NewPseudoWorkerPool(), + localPool: util.NewPseudoWorkerPool(), + log: &logger.Logger{Logger: zap.L()}, + keyStorage: ks, + clientConstructor: cc, + maxSizeSrc: ms, + localStore: os, + cnrSrc: cs, + netMapSrc: ns, + netmapKeys: nk, + networkState: nst, } -} - -func NewService(opts ...Option) *Service { - c := defaultCfg() for i := range opts { opts[i](c) } - c.fmtValidator = object.NewFormatValidator(c.fmtValidatorOpts...) + c.fmtValidator = object.NewFormatValidator(object.WithLockSource(os), object.WithNetState(nst)) return &Service{ cfg: c, @@ -83,62 +93,12 @@ func (p *Service) Put() (*Streamer, error) { }, nil } -func WithKeyStorage(v *objutil.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = v - } -} - -func WithMaxSizeSource(v MaxSizeSource) Option { - return func(c *cfg) { - c.maxSizeSrc = v - } -} - -func WithObjectStorage(v ObjectStorage) Option { - return func(c *cfg) { - c.localStore = v - c.fmtValidatorOpts = append(c.fmtValidatorOpts, object.WithLockSource(v)) - } -} - -func WithContainerSource(v container.Source) Option { - return func(c *cfg) { - c.cnrSrc = v - } -} - -func WithNetworkMapSource(v netmap.Source) Option { - return func(c *cfg) { - c.netMapSrc = v - } -} - func WithWorkerPools(remote, local util.WorkerPool) Option { return func(c *cfg) { c.remotePool, c.localPool = remote, local } } -func WithNetmapKeys(v netmap.AnnouncedKeys) Option { - return func(c *cfg) { - c.netmapKeys = v - } -} - -func WithNetworkState(v netmap.State) Option { - return func(c *cfg) { - c.networkState = v - c.fmtValidatorOpts = append(c.fmtValidatorOpts, object.WithNetState(v)) - } -} - -func WithClientConstructor(v ClientConstructor) Option { - return func(c *cfg) { - c.clientConstructor = v - } -} - func WithLogger(l *logger.Logger) Option { return func(c *cfg) { c.log = l diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go new file mode 100644 index 000000000..200830e15 --- /dev/null +++ b/pkg/services/object/put/single.go @@ -0,0 +1,369 @@ +package putsvc + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "hash" + "sync" + "sync/atomic" + + objectAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/tzhash/tz" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +type putSingleRequestSigner struct { + req *objectAPI.PutSingleRequest + keyStorage *svcutil.KeyStorage + signer *sync.Once +} + +func (s *putSingleRequestSigner) GetRequestWithSignedHeader() (*objectAPI.PutSingleRequest, error) { + var resErr error + s.signer.Do(func() { + metaHdr := new(sessionV2.RequestMetaHeader) + meta := s.req.GetMetaHeader() + + metaHdr.SetTTL(meta.GetTTL() - 1) + metaHdr.SetOrigin(meta) + s.req.SetMetaHeader(metaHdr) + + privateKey, err := s.keyStorage.GetKey(nil) + if err != nil { + resErr = err + return + } + resErr = signature.SignServiceMessage(privateKey, s.req) + }) + return s.req, resErr +} + +func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "putsvc.PutSingle") + defer span.End() + + obj := objectSDK.NewFromV2(req.GetBody().GetObject()) + + if err := s.validatePutSingle(ctx, obj); err != nil { + return nil, err + } + + if err := s.saveToNodes(ctx, obj, req); err != nil { + return nil, err + } + + resp := &objectAPI.PutSingleResponse{} + resp.SetBody(&objectAPI.PutSingleResponseBody{}) + return resp, nil +} + +func (s *Service) validatePutSingle(ctx context.Context, obj *objectSDK.Object) error { + if err := s.validarePutSingleSize(obj); err != nil { + return err + } + + if err := s.validatePutSingleChecksum(obj); err != nil { + return err + } + + return s.validatePutSingleObject(ctx, obj) +} + +func (s *Service) validarePutSingleSize(obj *objectSDK.Object) error { + if uint64(len(obj.Payload())) != obj.PayloadSize() { + return ErrWrongPayloadSize + } + + maxAllowedSize := s.maxSizeSrc.MaxObjectSize() + if obj.PayloadSize() > maxAllowedSize { + return ErrExceedingMaxSize + } + + return nil +} + +func (s *Service) validatePutSingleChecksum(obj *objectSDK.Object) error { + cs, csSet := obj.PayloadChecksum() + if !csSet { + return errors.New("missing payload checksum") + } + + var hash hash.Hash + + switch typ := cs.Type(); typ { + default: + return fmt.Errorf("unsupported payload checksum type %v", typ) + case checksum.SHA256: + hash = sha256.New() + case checksum.TZ: + hash = tz.New() + } + + if _, err := hash.Write(obj.Payload()); err != nil { + return fmt.Errorf("could not compute payload hash: %w", err) + } + + if !bytes.Equal(hash.Sum(nil), cs.Value()) { + return fmt.Errorf("incorrect payload checksum") + } + + return nil +} + +func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Object) error { + if err := s.fmtValidator.Validate(ctx, obj, false); err != nil { + return fmt.Errorf("coud not validate object format: %w", err) + } + + _, err := s.fmtValidator.ValidateContent(obj) + if err != nil { + return fmt.Errorf("could not validate payload content: %w", err) + } + + return nil +} + +func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *objectAPI.PutSingleRequest) error { + localOnly := req.GetMetaHeader().GetTTL() <= 1 + placementOptions, err := s.getPutSinglePlacementOptions(obj, req.GetBody().GetCopiesNumber(), localOnly) + if err != nil { + return err + } + traversal := &traversal{ + opts: placementOptions, + extraBroadcastEnabled: len(obj.Children()) > 0 || + (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock)), + mtx: sync.RWMutex{}, + mExclude: make(map[string]struct{}), + } + signer := &putSingleRequestSigner{ + req: req, + keyStorage: s.keyStorage, + signer: &sync.Once{}, + } + return s.saveAccordingToPlacement(ctx, obj, signer, traversal) +} + +func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) ([]placement.Option, error) { + var result []placement.Option + if len(copiesNumber) > 0 { + result = append(result, placement.WithCopyNumbers(copiesNumber)) + } + + cnrID, ok := obj.ContainerID() + if !ok { + return nil, errors.New("missing container ID") + } + cnrInfo, err := s.cnrSrc.Get(cnrID) + if err != nil { + return nil, fmt.Errorf("could not get container by ID: %w", err) + } + result = append(result, placement.ForContainer(cnrInfo.Value)) + + objID, ok := obj.ID() + if !ok { + return nil, errors.New("missing object ID") + } + result = append(result, placement.ForObject(objID)) + + latestNetmap, err := netmap.GetLatestNetworkMap(s.netMapSrc) + if err != nil { + return nil, fmt.Errorf("could not get latest network map: %w", err) + } + builder := placement.NewNetworkMapBuilder(latestNetmap) + if localOnly { + result = append(result, placement.SuccessAfter(1)) + builder = svcutil.NewLocalPlacement(builder, s.netmapKeys) + } + result = append(result, placement.UseBuilder(builder)) + return result, nil +} + +func (s *Service) saveAccordingToPlacement(ctx context.Context, obj *objectSDK.Object, signer *putSingleRequestSigner, traversal *traversal) error { + traverser, err := placement.NewTraverser(traversal.opts...) + if err != nil { + return fmt.Errorf("could not create object placement traverser: %w", err) + } + + var resultError atomic.Value + for { + addrs := traverser.Next() + if len(addrs) == 0 { + break + } + + if stop := s.saveToPlacementNodes(ctx, obj, signer, traversal, traverser, addrs, &resultError); stop { + break + } + } + + if !traverser.Success() { + var err errIncompletePut + err.singleErr, _ = resultError.Load().(error) + return err + } + + if traversal.submitPrimaryPlacementFinish() { + err = s.saveAccordingToPlacement(ctx, obj, signer, traversal) + if err != nil { + s.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) + } + } + + return nil +} + +func (s *Service) saveToPlacementNodes(ctx context.Context, + obj *objectSDK.Object, + signer *putSingleRequestSigner, + traversal *traversal, + traverser *placement.Traverser, + nodeAddresses []placement.Node, + resultError *atomic.Value, +) bool { + wg := sync.WaitGroup{} + + for _, nodeAddress := range nodeAddresses { + nodeAddress := nodeAddress + if traversal.processed(nodeAddress) { + continue + } + + local := false + workerPool := s.remotePool + if s.netmapKeys.IsLocalKey(nodeAddress.PublicKey()) { + local = true + workerPool = s.localPool + } + + wg.Add(1) + if err := workerPool.Submit(func() { + defer wg.Done() + + err := s.saveToPlacementNode(ctx, &nodeDesc{local: local, info: nodeAddress}, obj, signer) + + traversal.submitProcessed(nodeAddress) + + if err != nil { + resultError.Store(err) + svcutil.LogServiceError(s.log, "PUT", nodeAddress.Addresses(), err) + return + } + + traverser.SubmitSuccess() + }); err != nil { + wg.Done() + svcutil.LogWorkerPoolError(s.log, "PUT", err) + return true + } + } + + wg.Wait() + + return false +} + +func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *nodeDesc, obj *objectSDK.Object, signer *putSingleRequestSigner) error { + if nodeDesc.local { + return s.localStore.Put(ctx, obj) + } + + var info client.NodeInfo + + client.NodeInfoFromNetmapElement(&info, nodeDesc.info) + + c, err := s.clientConstructor.Get(info) + if err != nil { + return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) + } + + return s.redirectPutSingleRequest(ctx, signer, obj, info, c) +} + +func (s *Service) redirectPutSingleRequest(ctx context.Context, + signer *putSingleRequestSigner, + obj *objectSDK.Object, + info client.NodeInfo, + c client.MultiAddressClient) error { + ctx, span := tracing.StartSpanFromContext(ctx, "putService.redirectPutSingleRequest") + defer span.End() + + var req *objectAPI.PutSingleRequest + var firstErr error + req, firstErr = signer.GetRequestWithSignedHeader() + if firstErr != nil { + return firstErr + } + + info.AddressGroup().IterateAddresses(func(addr network.Address) (stop bool) { + ctx, span := tracing.StartSpanFromContext(ctx, "putService.redirectPutSingleRequest.IterateAddresses", + trace.WithAttributes( + attribute.String("address", addr.String()), + )) + defer span.End() + + var err error + + defer func() { + if err != nil { + objID, _ := obj.ID() + cnrID, _ := obj.ContainerID() + s.log.Warn("failed to redirect PutSingle request", + zap.Error(err), + zap.Stringer("address", addr), + zap.Stringer("object_id", objID), + zap.Stringer("container_id", cnrID), + ) + } + + stop = err == nil + if stop || firstErr == nil { + firstErr = err + } + }() + + var resp *objectAPI.PutSingleResponse + + err = c.RawForAddress(ctx, addr, func(cli *rawclient.Client) error { + var e error + resp, e = rpc.PutSingleObject(cli, req, rawclient.WithContext(ctx)) + return e + }) + if err != nil { + err = fmt.Errorf("failed to execute request: %w", err) + return + } + + if err = internal.VerifyResponseKeyV2(info.PublicKey(), resp); err != nil { + return + } + + err = signature.VerifyServiceMessage(resp) + if err != nil { + err = fmt.Errorf("response verification failed: %w", err) + } + + return + }) + + return firstErr +} diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 6d0d8062e..80b1c2541 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -10,8 +10,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -21,7 +22,7 @@ type Streamer struct { sessionKey *ecdsa.PrivateKey - target transformer.ObjectTarget + target transformer.ChunkedObjectWriter relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error @@ -128,7 +129,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { fmt: p.fmtValidator, nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ Key: sessionKey, - NextTargetInit: func() transformer.ObjectTarget { return p.newCommonTarget(prm) }, + NextTargetInit: func() transformer.ObjectWriter { return p.newCommonTarget(prm) }, NetworkState: p.networkState, MaxSize: p.maxPayloadSz, WithoutHomomorphicHash: containerSDK.IsHomomorphicHashingDisabled(prm.cnr), @@ -191,7 +192,7 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error { return nil } -func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { +func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { var relay func(context.Context, nodeDesc) error if p.relay != nil { relay = func(ctx context.Context, node nodeDesc) error { @@ -211,7 +212,7 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { // enable additional container broadcast on non-local operation // if object has TOMBSTONE or LOCK type. typ := prm.hdr.Type() - withBroadcast := !prm.common.LocalOnly() && (typ == object.TypeTombstone || typ == object.TypeLock) + withBroadcast := !prm.common.LocalOnly() && (typ == objectSDK.TypeTombstone || typ == objectSDK.TypeLock) return &distributedTarget{ traversal: traversal{ @@ -219,12 +220,11 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { extraBroadcastEnabled: withBroadcast, }, - payload: getPayload(), - remotePool: p.remotePool, - localPool: p.localPool, + payload: getPayload(), + getWorkerPool: p.getWorkerPool, nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { - return &localTarget{ + return localTarget{ storage: p.localStore, } } @@ -242,8 +242,6 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { relay: relay, fmt: p.fmtValidator, log: p.log, - - isLocalKey: p.netmapKeys.IsLocalKey, } } @@ -280,3 +278,10 @@ func (p *Streamer) Close(ctx context.Context) (*PutResponse, error) { id: ids.SelfID, }, nil } + +func (p *Streamer) getWorkerPool(pub []byte) (pkgutil.WorkerPool, bool) { + if p.netmapKeys.IsLocalKey(pub) { + return p.localPool, true + } + return p.remotePool, false +} diff --git a/pkg/services/object/put/v2/service.go b/pkg/services/object/put/v2/service.go index 656f8df9c..db902ae59 100644 --- a/pkg/services/object/put/v2/service.go +++ b/pkg/services/object/put/v2/service.go @@ -1,8 +1,10 @@ package putsvc import ( + "context" "fmt" + objectAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -10,27 +12,15 @@ import ( // Service implements Put operation of Object service v2. type Service struct { - *cfg -} - -// Option represents Service constructor option. -type Option func(*cfg) - -type cfg struct { svc *putsvc.Service keyStorage *util.KeyStorage } // NewService constructs Service instance from provided options. -func NewService(opts ...Option) *Service { - c := new(cfg) - - for i := range opts { - opts[i](c) - } - +func NewService(svc *putsvc.Service, ks *util.KeyStorage) *Service { return &Service{ - cfg: c, + svc: svc, + keyStorage: ks, } } @@ -47,14 +37,6 @@ func (s *Service) Put() (object.PutObjectStream, error) { }, nil } -func WithInternalService(v *putsvc.Service) Option { - return func(c *cfg) { - c.svc = v - } -} - -func WithKeyStorage(ks *util.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = ks - } +func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { + return s.svc.PutSingle(ctx, req) } diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 65531dc66..9c6de4ca8 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -5,7 +5,6 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -16,6 +15,7 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/services/object/put/v2/util.go b/pkg/services/object/put/v2/util.go index 758470f6c..a157a9542 100644 --- a/pkg/services/object/put/v2/util.go +++ b/pkg/services/object/put/v2/util.go @@ -5,7 +5,7 @@ import ( refsV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) func (s *streamer) toInitPrm(part *objectV2.PutObjectPartInit, req *objectV2.PutRequest) (*putsvc.PutInitPrm, error) { @@ -21,7 +21,7 @@ func (s *streamer) toInitPrm(part *objectV2.PutObjectPartInit, req *objectV2.Put return new(putsvc.PutInitPrm). WithObject( - object.NewFromV2(oV2), + objectSDK.NewFromV2(oV2), ). WithRelay(s.relayRequest). WithCommonPrm(commonPrm). diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index 406304422..c2b078ef5 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -17,14 +17,14 @@ import ( // validatingTarget validates unprepared object format and content (streaming PUT case). type validatingTarget struct { - nextTarget transformer.ObjectTarget + nextTarget transformer.ChunkedObjectWriter fmt *object.FormatValidator } // validatingPreparedTarget validates prepared object format and content. type validatingPreparedTarget struct { - nextTarget transformer.ObjectTarget + nextTarget transformer.ChunkedObjectWriter fmt *object.FormatValidator @@ -48,7 +48,7 @@ var ( func (t *validatingTarget) WriteHeader(ctx context.Context, obj *objectSDK.Object) error { if err := t.fmt.Validate(ctx, obj, true); err != nil { - return fmt.Errorf("(%T) coult not validate object format: %w", t, err) + return fmt.Errorf("(%T) could not validate object format: %w", t, err) } return t.nextTarget.WriteHeader(ctx, obj) @@ -93,7 +93,7 @@ func (t *validatingPreparedTarget) WriteHeader(ctx context.Context, obj *objectS t.checksum = cs.Value() if err := t.fmt.Validate(ctx, obj, false); err != nil { - return fmt.Errorf("(%T) coult not validate object format: %w", t, err) + return fmt.Errorf("(%T) could not validate object format: %w", t, err) } err := t.nextTarget.WriteHeader(ctx, obj) diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index def934ea6..a10f26a34 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -5,7 +5,6 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" ) @@ -16,25 +15,26 @@ type ResponseService struct { } type searchStreamResponser struct { - util.ServerStream + SearchStream - respWriter util.ResponseMessageWriter + respSvc *response.Service } type getStreamResponser struct { - util.ServerStream + GetObjectStream - respWriter util.ResponseMessageWriter + respSvc *response.Service } type getRangeStreamResponser struct { - util.ServerStream + GetObjectRangeStream - respWriter util.ResponseMessageWriter + respSvc *response.Service } type putStreamResponser struct { - stream *response.ClientMessageStreamer + stream PutObjectStream + respSvc *response.Service } // NewResponseService returns object service instance that passes internal service @@ -47,29 +47,32 @@ func NewResponseService(objSvc ServiceServer, respSvc *response.Service) *Respon } func (s *getStreamResponser) Send(resp *object.GetResponse) error { - return s.respWriter(resp) + s.respSvc.SetMeta(resp) + return s.GetObjectStream.Send(resp) } func (s *ResponseService) Get(req *object.GetRequest, stream GetObjectStream) error { return s.svc.Get(req, &getStreamResponser{ - ServerStream: stream, - respWriter: s.respSvc.HandleServerStreamRequest(func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetResponse)) - }), + GetObjectStream: stream, + respSvc: s.respSvc, }) } func (s *putStreamResponser) Send(ctx context.Context, req *object.PutRequest) error { - return s.stream.Send(ctx, req) + if err := s.stream.Send(ctx, req); err != nil { + return fmt.Errorf("could not send the request: %w", err) + } + return nil } func (s *putStreamResponser) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { r, err := s.stream.CloseAndRecv(ctx) if err != nil { - return nil, fmt.Errorf("(%T) could not receive response: %w", s, err) + return nil, fmt.Errorf("could not close stream and receive response: %w", err) } - return r.(*object.PutResponse), nil + s.respSvc.SetMeta(r) + return r, nil } func (s *ResponseService) Put() (PutObjectStream, error) { @@ -79,78 +82,71 @@ func (s *ResponseService) Put() (PutObjectStream, error) { } return &putStreamResponser{ - stream: s.respSvc.CreateRequestStreamer( - func(ctx context.Context, req any) error { - return stream.Send(ctx, req.(*object.PutRequest)) - }, - func(ctx context.Context) (util.ResponseMessage, error) { - return stream.CloseAndRecv(ctx) - }, - ), + stream: stream, + respSvc: s.respSvc, }, nil } -func (s *ResponseService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Head(ctx, req.(*object.HeadRequest)) - }, - ) +func (s *ResponseService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + resp, err := s.svc.PutSingle(ctx, req) if err != nil { return nil, err } - return resp.(*object.HeadResponse), nil + s.respSvc.SetMeta(resp) + return resp, nil +} + +func (s *ResponseService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { + resp, err := s.svc.Head(ctx, req) + if err != nil { + return nil, err + } + + s.respSvc.SetMeta(resp) + return resp, nil } func (s *searchStreamResponser) Send(resp *object.SearchResponse) error { - return s.respWriter(resp) + s.respSvc.SetMeta(resp) + return s.SearchStream.Send(resp) } func (s *ResponseService) Search(req *object.SearchRequest, stream SearchStream) error { return s.svc.Search(req, &searchStreamResponser{ - ServerStream: stream, - respWriter: s.respSvc.HandleServerStreamRequest(func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.SearchResponse)) - }), + SearchStream: stream, + respSvc: s.respSvc, }) } func (s *ResponseService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*object.DeleteRequest)) - }, - ) + resp, err := s.svc.Delete(ctx, req) if err != nil { return nil, err } - return resp.(*object.DeleteResponse), nil + s.respSvc.SetMeta(resp) + return resp, nil } func (s *getRangeStreamResponser) Send(resp *object.GetRangeResponse) error { - return s.respWriter(resp) + s.respSvc.SetMeta(resp) + return s.GetObjectRangeStream.Send(resp) } func (s *ResponseService) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { return s.svc.GetRange(req, &getRangeStreamResponser{ - ServerStream: stream, - respWriter: s.respSvc.HandleServerStreamRequest(func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetRangeResponse)) - }), + GetObjectRangeStream: stream, + respSvc: s.respSvc, }) } func (s *ResponseService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetRangeHash(ctx, req.(*object.GetRangeHashRequest)) - }, - ) + resp, err := s.svc.GetRangeHash(ctx, req) if err != nil { return nil, err } - return resp.(*object.GetRangeHashResponse), nil + s.respSvc.SetMeta(resp) + return resp, nil } diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index 475a31b98..c1a9a0c1c 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -5,7 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -56,7 +56,7 @@ func (exec *execCtx) containerID() cid.ID { return exec.prm.cnr } -func (exec *execCtx) searchFilters() object.SearchFilters { +func (exec *execCtx) searchFilters() objectSDK.SearchFilters { return exec.prm.filters } diff --git a/pkg/services/object/search/prm.go b/pkg/services/object/search/prm.go index da46dfeb6..d2918d6e7 100644 --- a/pkg/services/object/search/prm.go +++ b/pkg/services/object/search/prm.go @@ -6,7 +6,7 @@ import ( coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -18,7 +18,7 @@ type Prm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters forwarder RequestForwarder } @@ -55,6 +55,6 @@ func (p *Prm) WithContainerID(id cid.ID) { } // WithSearchFilters sets search filters. -func (p *Prm) WithSearchFilters(fs object.SearchFilters) { +func (p *Prm) WithSearchFilters(fs objectSDK.SearchFilters) { p.filters = fs } diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index 708979d79..16b82a620 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -55,17 +55,28 @@ type cfg struct { keyStore *util.KeyStorage } -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - clientConstructor: new(clientConstructorWrapper), - } -} - // New creates, initializes and returns utility serving // Object.Get service requests. -func New(opts ...Option) *Service { - c := defaultCfg() +func New(e *engine.StorageEngine, + cc ClientConstructor, + tg *util.TraverserGenerator, + ns netmap.Source, + ks *util.KeyStorage, + opts ...Option) *Service { + c := &cfg{ + log: &logger.Logger{Logger: zap.L()}, + clientConstructor: &clientConstructorWrapper{ + constructor: cc, + }, + localStorage: &storageEngineWrapper{ + storage: e, + }, + traverserGenerator: (*traverseGeneratorWrapper)(tg), + currentEpochReceiver: &nmSrcWrapper{ + nmSrc: ns, + }, + keyStore: ks, + } for i := range opts { opts[i](c) @@ -82,46 +93,3 @@ func WithLogger(l *logger.Logger) Option { c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Search service"))} } } - -// WithLocalStorageEngine returns option to set local storage -// instance. -func WithLocalStorageEngine(e *engine.StorageEngine) Option { - return func(c *cfg) { - c.localStorage = &storageEngineWrapper{ - storage: e, - } - } -} - -// WithClientConstructor returns option to set constructor of remote node clients. -func WithClientConstructor(v ClientConstructor) Option { - return func(c *cfg) { - c.clientConstructor.(*clientConstructorWrapper).constructor = v - } -} - -// WithTraverserGenerator returns option to set generator of -// placement traverser to get the objects from containers. -func WithTraverserGenerator(t *util.TraverserGenerator) Option { - return func(c *cfg) { - c.traverserGenerator = (*traverseGeneratorWrapper)(t) - } -} - -// WithNetMapSource returns option to set network -// map storage to receive current network state. -func WithNetMapSource(nmSrc netmap.Source) Option { - return func(c *cfg) { - c.currentEpochReceiver = &nmSrcWrapper{ - nmSrc: nmSrc, - } - } -} - -// WithKeyStorage returns option to set private -// key storage for session tokens and node key. -func WithKeyStorage(store *util.KeyStorage) Option { - return func(c *cfg) { - c.keyStore = store - } -} diff --git a/pkg/services/object/search/v2/request_forwarder.go b/pkg/services/object/search/v2/request_forwarder.go index d8719986f..5a2e9b936 100644 --- a/pkg/services/object/search/v2/request_forwarder.go +++ b/pkg/services/object/search/v2/request_forwarder.go @@ -20,7 +20,7 @@ import ( ) type requestForwarder struct { - OnceResign *sync.Once + OnceResign sync.Once Request *objectV2.SearchRequest Key *ecdsa.PrivateKey } diff --git a/pkg/services/object/search/v2/service.go b/pkg/services/object/search/v2/service.go index 17e1bc7e0..78b72ac79 100644 --- a/pkg/services/object/search/v2/service.go +++ b/pkg/services/object/search/v2/service.go @@ -9,28 +9,15 @@ import ( // Service implements Search operation of Object service v2. type Service struct { - *cfg -} - -// Option represents Service constructor option. -type Option func(*cfg) - -type cfg struct { - svc *searchsvc.Service - + svc *searchsvc.Service keyStorage *objutil.KeyStorage } // NewService constructs Service instance from provided options. -func NewService(opts ...Option) *Service { - c := new(cfg) - - for i := range opts { - opts[i](c) - } - +func NewService(s *searchsvc.Service, ks *objutil.KeyStorage) *Service { return &Service{ - cfg: c, + svc: s, + keyStorage: ks, } } @@ -43,18 +30,3 @@ func (s *Service) Search(req *objectV2.SearchRequest, stream objectSvc.SearchStr return s.svc.Search(stream.Context(), *p) } - -// WithInternalService returns option to set entity -// that handles request payload. -func WithInternalService(v *searchsvc.Service) Option { - return func(c *cfg) { - c.svc = v - } -} - -// WithKeyStorage returns option to set local private key storage. -func WithKeyStorage(ks *objutil.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = ks - } -} diff --git a/pkg/services/object/search/v2/util.go b/pkg/services/object/search/v2/util.go index 12158a820..e971fa8e5 100644 --- a/pkg/services/object/search/v2/util.go +++ b/pkg/services/object/search/v2/util.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" @@ -13,7 +12,7 @@ import ( searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -51,16 +50,15 @@ func (s *Service) toPrm(req *objectV2.SearchRequest, stream objectSvc.SearchStre } forwarder := &requestForwarder{ - OnceResign: &sync.Once{}, - Request: req, - Key: key, + Request: req, + Key: key, } p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequest)) } p.WithContainerID(id) - p.WithSearchFilters(object.NewSearchFiltersFromV2(body.GetFilters())) + p.WithSearchFilters(objectSDK.NewSearchFiltersFromV2(body.GetFilters())) return p, nil } diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index ccce9c4f4..73b88f233 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -41,4 +41,5 @@ type ServiceServer interface { Delete(context.Context, *object.DeleteRequest) (*object.DeleteResponse, error) GetRange(*object.GetRangeRequest, GetObjectRangeStream) error GetRangeHash(context.Context, *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) + PutSingle(context.Context, *object.PutSingleRequest) (*object.PutSingleResponse, error) } diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 9d66c76ba..c516872e3 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -18,27 +18,30 @@ type SignService struct { } type searchStreamSigner struct { - util.ServerStream - - respWriter util.ResponseMessageWriter + SearchStream + statusSupported bool + sigSvc *util.SignService nonEmptyResp bool // set on first Send call } type getStreamSigner struct { - util.ServerStream - - respWriter util.ResponseMessageWriter + GetObjectStream + statusSupported bool + sigSvc *util.SignService } type putStreamSigner struct { - stream *util.RequestMessageStreamer + sigSvc *util.SignService + stream PutObjectStream + statusSupported bool + err error } type getRangeStreamSigner struct { - util.ServerStream - - respWriter util.ResponseMessageWriter + GetObjectRangeStream + statusSupported bool + sigSvc *util.SignService } func NewSignService(key *ecdsa.PrivateKey, svc ServiceServer) *SignService { @@ -50,37 +53,50 @@ func NewSignService(key *ecdsa.PrivateKey, svc ServiceServer) *SignService { } func (s *getStreamSigner) Send(resp *object.GetResponse) error { - return s.respWriter(resp) + if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + return err + } + return s.GetObjectStream.Send(resp) } func (s *SignService) Get(req *object.GetRequest, stream GetObjectStream) error { - return s.sigSvc.HandleServerStreamRequest(req, - func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetResponse)) - }, - func() util.ResponseMessage { - return new(object.GetResponse) - }, - func(respWriter util.ResponseMessageWriter) error { - return s.svc.Get(req, &getStreamSigner{ - ServerStream: stream, - respWriter: respWriter, - }) - }, - ) + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.GetResponse) + _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return stream.Send(resp) + } + + return s.svc.Get(req, &getStreamSigner{ + GetObjectStream: stream, + sigSvc: s.sigSvc, + statusSupported: util.IsStatusSupported(req), + }) } func (s *putStreamSigner) Send(ctx context.Context, req *object.PutRequest) error { - return s.stream.Send(ctx, req) + s.statusSupported = util.IsStatusSupported(req) + + if s.err = s.sigSvc.VerifyRequest(req); s.err != nil { + return util.ErrAbortStream + } + if s.err = s.stream.Send(ctx, req); s.err != nil { + return util.ErrAbortStream + } + return nil } -func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { - r, err := s.stream.CloseAndRecv(ctx) - if err != nil { - return nil, fmt.Errorf("could not receive response: %w", err) +func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.PutResponse, err error) { + if s.err != nil { + err = s.err + resp = new(object.PutResponse) + } else { + resp, err = s.stream.CloseAndRecv(ctx) + if err != nil { + return nil, fmt.Errorf("could not close stream and receive response: %w", err) + } } - return r.(*object.PutResponse), nil + return resp, s.sigSvc.SignResponse(s.statusSupported, resp, err) } func (s *SignService) Put() (PutObjectStream, error) { @@ -90,120 +106,96 @@ func (s *SignService) Put() (PutObjectStream, error) { } return &putStreamSigner{ - stream: s.sigSvc.CreateRequestStreamer( - func(ctx context.Context, req any) error { - return stream.Send(ctx, req.(*object.PutRequest)) - }, - func(ctx context.Context) (util.ResponseMessage, error) { - return stream.CloseAndRecv(ctx) - }, - func() util.ResponseMessage { - return new(object.PutResponse) - }, - ), + stream: stream, + sigSvc: s.sigSvc, }, nil } func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Head(ctx, req.(*object.HeadRequest)) - }, - func() util.ResponseMessage { - return new(object.HeadResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.HeadResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } + resp, err := util.EnsureNonNilResponse(s.svc.Head(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) +} - return resp.(*object.HeadResponse), nil +func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.PutSingleResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + } + resp, err := util.EnsureNonNilResponse(s.svc.PutSingle(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { s.nonEmptyResp = true - return s.respWriter(resp) + if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + return err + } + return s.SearchStream.Send(resp) } func (s *SignService) Search(req *object.SearchRequest, stream SearchStream) error { - return s.sigSvc.HandleServerStreamRequest(req, - func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.SearchResponse)) - }, - func() util.ResponseMessage { - return new(object.SearchResponse) - }, - func(respWriter util.ResponseMessageWriter) error { - stream := &searchStreamSigner{ - ServerStream: stream, - respWriter: respWriter, - } + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.SearchResponse) + _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return stream.Send(resp) + } - err := s.svc.Search(req, stream) - - if err == nil && !stream.nonEmptyResp { - // The higher component does not write any response in the case of an empty result (which is correct). - // With the introduction of status returns at least one answer must be signed and sent to the client. - // This approach is supported by clients who do not know how to work with statuses (one could make - // a switch according to the protocol version from the request, but the costs of sending an empty - // answer can be neglected due to the gradual refusal to use the "old" clients). - return stream.Send(new(object.SearchResponse)) - } - - return err - }, - ) + ss := &searchStreamSigner{ + SearchStream: stream, + sigSvc: s.sigSvc, + statusSupported: util.IsStatusSupported(req), + } + err := s.svc.Search(req, ss) + if err == nil && !ss.nonEmptyResp { + // The higher component does not write any response in the case of an empty result (which is correct). + // With the introduction of status returns at least one answer must be signed and sent to the client. + // This approach is supported by clients who do not know how to work with statuses (one could make + // a switch according to the protocol version from the request, but the costs of sending an empty + // answer can be neglected due to the gradual refusal to use the "old" clients). + return stream.Send(new(object.SearchResponse)) + } + return err } func (s *SignService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*object.DeleteRequest)) - }, - func() util.ResponseMessage { - return new(object.DeleteResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.DeleteResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*object.DeleteResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *getRangeStreamSigner) Send(resp *object.GetRangeResponse) error { - return s.respWriter(resp) + if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + return err + } + return s.GetObjectRangeStream.Send(resp) } func (s *SignService) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { - return s.sigSvc.HandleServerStreamRequest(req, - func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetRangeResponse)) - }, - func() util.ResponseMessage { - return new(object.GetRangeResponse) - }, - func(respWriter util.ResponseMessageWriter) error { - return s.svc.GetRange(req, &getRangeStreamSigner{ - ServerStream: stream, - respWriter: respWriter, - }) - }, - ) + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.GetRangeResponse) + _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return stream.Send(resp) + } + + return s.svc.GetRange(req, &getRangeStreamSigner{ + GetObjectRangeStream: stream, + sigSvc: s.sigSvc, + statusSupported: util.IsStatusSupported(req), + }) } func (s *SignService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetRangeHash(ctx, req.(*object.GetRangeHashRequest)) - }, - func() util.ResponseMessage { - return new(object.GetRangeHashResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.GetRangeHashResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*object.GetRangeHashResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.GetRangeHash(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index a7d1c486a..2d9810cd3 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -107,6 +107,10 @@ func (c TransportSplitter) Delete(ctx context.Context, request *object.DeleteReq return c.next.Delete(ctx, request) } +func (c TransportSplitter) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + return c.next.PutSingle(ctx, req) +} + func (s *rangeStreamMsgSizeCtrl) Send(resp *object.GetRangeResponse) error { body := resp.GetBody() diff --git a/pkg/services/object/util/chain.go b/pkg/services/object/util/chain.go index 96dafd10e..b574d5eb6 100644 --- a/pkg/services/object/util/chain.go +++ b/pkg/services/object/util/chain.go @@ -5,7 +5,7 @@ import ( "fmt" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -22,11 +22,11 @@ type HeadReceiver interface { // // If reverseDirection arg is true, then the traversal is done in reverse order. // Stop boolean result provides the ability to interrupt the traversal. -type SplitMemberHandler func(member *object.Object, reverseDirection bool) (stop bool) +type SplitMemberHandler func(member *objectSDK.Object, reverseDirection bool) (stop bool) // IterateAllSplitLeaves is an iterator over all object split-tree leaves in direct order. -func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Object)) error { - return IterateSplitLeaves(r, addr, func(leaf *object.Object) bool { +func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*objectSDK.Object)) error { + return IterateSplitLeaves(r, addr, func(leaf *objectSDK.Object) bool { h(leaf) return false }) @@ -35,13 +35,13 @@ func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Obje // IterateSplitLeaves is an iterator over object split-tree leaves in direct order. // // If member handler returns true, then the iterator aborts without error. -func IterateSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Object) bool) error { +func IterateSplitLeaves(r HeadReceiver, addr oid.Address, h func(*objectSDK.Object) bool) error { var ( reverse bool - leaves []*object.Object + leaves []*objectSDK.Object ) - if err := TraverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if err := TraverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { reverse = reverseDirection if reverse { @@ -84,9 +84,9 @@ func traverseSplitChain(r HeadReceiver, addr oid.Address, h SplitMemberHandler) switch res := v.(type) { default: panic(fmt.Sprintf("unexpected result of %T: %T", r, v)) - case *object.Object: + case *objectSDK.Object: return h(res, false), nil - case *object.SplitInfo: + case *objectSDK.SplitInfo: link, withLink := res.Link() last, withLast := res.LastPart() @@ -108,7 +108,7 @@ func traverseByLink(cnr cid.ID, link oid.ID, r HeadReceiver, h SplitMemberHandle chain := make([]oid.ID, 0) - if _, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if _, err := traverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { children := member.Children() if reverseDirection { @@ -122,12 +122,12 @@ func traverseByLink(cnr cid.ID, link oid.ID, r HeadReceiver, h SplitMemberHandle return false, err } - var reverseChain []*object.Object + var reverseChain []*objectSDK.Object for i := range chain { addr.SetObject(chain[i]) - if stop, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if stop, err := traverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { if !reverseDirection { return h(member, false) } @@ -147,16 +147,16 @@ func traverseByLink(cnr cid.ID, link oid.ID, r HeadReceiver, h SplitMemberHandle return false, nil } -func traverseByLast(cnr cid.ID, last oid.ID, withLast bool, res *object.SplitInfo, r HeadReceiver, h SplitMemberHandler) (bool, error) { +func traverseByLast(cnr cid.ID, last oid.ID, withLast bool, res *objectSDK.SplitInfo, r HeadReceiver, h SplitMemberHandler) (bool, error) { var addr oid.Address addr.SetContainer(cnr) for last, withLast = res.LastPart(); withLast; { addr.SetObject(last) - var directChain []*object.Object + var directChain []*objectSDK.Object - if _, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if _, err := traverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { if reverseDirection { last, withLast = member.PreviousID() return h(member, true) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index e46240a86..c59146e2b 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -29,7 +29,7 @@ type Option func(*cfg) // Traverser represents utility for controlling // traversal of object placement vectors. type Traverser struct { - mtx *sync.RWMutex + mtx sync.RWMutex vectors [][]netmap.NodeInfo @@ -86,7 +86,7 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } // backward compatibility for scalar `copies_number` - if len(cfg.copyNumbers) == 1 { + if len(cfg.copyNumbers) == 1 && cfg.copyNumbers[0] != 0 { cfg.flatSuccess = &cfg.copyNumbers[0] } @@ -97,17 +97,22 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } else { rem = defaultCopiesVector(cfg.policy) + // compatibleZeroVector is a bool flag which is set when cfg.copyNumbers + // is [0]. In this case we should not modify `rem` slice unless track + // copies are ignored, because [0] means that all copies should be + // stored before returning OK to the client. + compatibleZeroVector := len(cfg.copyNumbers) == 1 && cfg.copyNumbers[0] == 0 + for i := range rem { if !cfg.trackCopies { rem[i] = -1 - } else if len(cfg.copyNumbers) > i { + } else if len(cfg.copyNumbers) > i && !compatibleZeroVector { rem[i] = int(cfg.copyNumbers[i]) } } } return &Traverser{ - mtx: new(sync.RWMutex), rem: rem, vectors: ns, }, nil diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index 66fd8afe0..1b307da6f 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -208,3 +208,55 @@ func TestTraverserObjectScenarios(t *testing.T) { require.True(t, tr.Success()) }) } + +func TestTraverserRemValues(t *testing.T) { + selectors := []int{3, 4, 5} + replicas := []int{2, 3, 4} + + nodes, cnr := testPlacement(t, selectors, replicas) + nodesCopy := copyVectors(nodes) + + testCases := [...]struct { + name string + copyNumbers []uint32 + expectedRem []int + }{ + { + name: "zero copy numbers", + copyNumbers: []uint32{}, + expectedRem: replicas, + }, + { + name: "compatible zero copy numbers", + copyNumbers: []uint32{0}, + expectedRem: replicas, + }, + { + name: "copy numbers for all replicas", + copyNumbers: []uint32{1, 1, 1}, + expectedRem: []int{1, 1, 1}, + }, + { + name: "single copy numbers for multiple replicas", + copyNumbers: []uint32{1}, + expectedRem: []int{1}, // may be a bit unexpected + }, + { + name: "multiple copy numbers for multiple replicas", + copyNumbers: []uint32{1, 1}, + expectedRem: []int{1, 1, 4}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + tr, err := NewTraverser( + ForContainer(cnr), + UseBuilder(&testBuilder{vectors: nodesCopy}), + WithCopyNumbers(testCase.copyNumbers), + ) + require.NoError(t, err) + require.Equal(t, testCase.expectedRem, tr.rem) + }) + } +} diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index 46fcc9840..66a0ec7d3 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -7,7 +7,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" "go.uber.org/zap" @@ -21,7 +21,7 @@ type Source interface { // // Tombstone MUST return (nil, nil) if requested tombstone is // missing in the storage for the provided epoch. - Tombstone(ctx context.Context, a oid.Address, epoch uint64) (*object.Object, error) + Tombstone(ctx context.Context, a oid.Address, epoch uint64) (*objectSDK.Object, error) } // ExpirationChecker is a tombstone source wrapper. @@ -72,7 +72,7 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr return false } -func (g *ExpirationChecker) handleTS(addr string, ts *object.Object, reqEpoch uint64) bool { +func (g *ExpirationChecker) handleTS(addr string, ts *objectSDK.Object, reqEpoch uint64) bool { for _, atr := range ts.Attributes() { if atr.Key() == objectV2.SysAttributeExpEpoch || atr.Key() == objectV2.SysAttributeExpEpochNeoFS { epoch, err := strconv.ParseUint(atr.Value(), 10, 64) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index e91b8871b..a45c019eb 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -7,66 +7,15 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - headsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/head" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) -// tracks Policer's check progress. -type nodeCache map[uint64]bool - -func newNodeCache() *nodeCache { - m := make(map[uint64]bool) - return (*nodeCache)(&m) -} - -func (n *nodeCache) set(node netmap.NodeInfo, val bool) { - (*n)[node.Hash()] = val -} - -// submits storage node as a candidate to store the object replica in case of -// shortage. -func (n *nodeCache) submitReplicaCandidate(node netmap.NodeInfo) { - n.set(node, false) -} - -// submits storage node as a current object replica holder. -func (n *nodeCache) submitReplicaHolder(node netmap.NodeInfo) { - n.set(node, true) -} - -// processStatus returns current processing status of the storage node -// -// >0 if node does not currently hold the object -// 0 if node already holds the object -// <0 if node has not been processed yet -func (n *nodeCache) processStatus(node netmap.NodeInfo) int8 { - val, ok := (*n)[node.Hash()] - if !ok { - return -1 - } - - if val { - return 0 - } - - return 1 -} - -// SubmitSuccessfulReplication marks given storage node as a current object -// replica holder. -// -// SubmitSuccessfulReplication implements replicator.TaskResult. -func (n *nodeCache) SubmitSuccessfulReplication(node netmap.NodeInfo) { - n.submitReplicaHolder(node) -} - func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.AddressWithType) { addr := addrWithType.Address idCnr := addr.Container() @@ -79,11 +28,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add zap.String("error", err.Error()), ) if container.IsErrNotFound(err) { - var prm engine.InhumePrm - prm.MarkAsGarbage(addrWithType.Address) - prm.WithForceRemoval() - - _, err := p.jobQueue.localStorage.Inhume(ctx, prm) + err := p.buryFn(ctx, addrWithType.Address) if err != nil { p.log.Error(logs.PolicerCouldNotInhumeObjectWithMissingContainer, zap.Stringer("cid", idCnr), @@ -145,15 +90,14 @@ type placementRequirements struct { } func (p *Policer) processNodes(ctx context.Context, requirements *placementRequirements, addrWithType objectcore.AddressWithType, - nodes []netmap.NodeInfo, shortage uint32, checkedNodes *nodeCache) { + nodes []netmap.NodeInfo, shortage uint32, checkedNodes nodeCache) { addr := addrWithType.Address typ := addrWithType.Type - prm := new(headsvc.RemoteHeadPrm).WithObjectAddress(addr) // Number of copies that are stored on maintenance nodes. var uncheckedCopies int - if typ == object.TypeLock { + if typ == objectSDK.TypeLock { // all nodes of a container must store the `LOCK` objects // for correct object removal protection: // - `LOCK` objects are broadcast on their PUT requests; @@ -175,8 +119,8 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi } else if nodes[i].IsMaintenance() { shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) } else { - if status := checkedNodes.processStatus(nodes[i]); status >= 0 { - if status == 0 { + if status := checkedNodes.processStatus(nodes[i]); status.Processed() { + if status == nodeHoldsObject { // node already contains replica, no need to replicate nodes = append(nodes[:i], nodes[i+1:]...) i-- @@ -188,7 +132,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi callCtx, cancel := context.WithTimeout(ctx, p.headTimeout) - _, err := p.remoteHeader.Head(callCtx, prm.WithNodeInfo(nodes[i])) + _, err := p.remoteHeader(callCtx, nodes[i], addr) cancel() @@ -224,7 +168,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi // prevent spam with new replicas. // However, additional copies should not be removed in this case, // because we can remove the only copy this way. -func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes *nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { +func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { checkedNodes.submitReplicaHolder(node) shortage-- uncheckedCopies++ @@ -236,25 +180,29 @@ func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes *nodeCach } func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address, requirements *placementRequirements, - nodes []netmap.NodeInfo, checkedNodes *nodeCache, shortage uint32, uncheckedCopies int) { - if shortage > 0 { + nodes []netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) { + switch { + case shortage > 0: p.log.Debug(logs.PolicerShortageOfObjectCopiesDetected, zap.Stringer("object", addr), zap.Uint32("shortage", shortage), ) - var task replicator.Task - task.SetObjectAddress(addr) - task.SetNodes(nodes) - task.SetCopiesNumber(shortage) + task := replicator.Task{ + NumCopies: shortage, + Addr: addr, + Nodes: nodes, + } p.replicator.HandleTask(ctx, task, checkedNodes) - } else if uncheckedCopies > 0 { + + case uncheckedCopies > 0: // If we have more copies than needed, but some of them are from the maintenance nodes, // save the local copy. p.log.Debug(logs.PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance, zap.Int("count", uncheckedCopies)) - } else if uncheckedCopies == 0 { + + case uncheckedCopies == 0: // Safe to remove: checked all copies, shortage == 0. requirements.removeLocalCopy = true } diff --git a/pkg/services/policer/check_test.go b/pkg/services/policer/check_test.go index b40ee90d2..d4c7ccbf9 100644 --- a/pkg/services/policer/check_test.go +++ b/pkg/services/policer/check_test.go @@ -11,14 +11,14 @@ func TestNodeCache(t *testing.T) { cache := newNodeCache() node := netmaptest.NodeInfo() - require.Negative(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeNotProcessed) cache.SubmitSuccessfulReplication(node) - require.Zero(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeHoldsObject) cache.submitReplicaCandidate(node) - require.Positive(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeDoesNotHoldObject) cache.submitReplicaHolder(node) - require.Zero(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeHoldsObject) } diff --git a/pkg/services/policer/nodecache.go b/pkg/services/policer/nodecache.go new file mode 100644 index 000000000..cd47cb0fc --- /dev/null +++ b/pkg/services/policer/nodecache.go @@ -0,0 +1,57 @@ +package policer + +import "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + +type nodeProcessStatus int8 + +const ( + nodeNotProcessed nodeProcessStatus = iota + nodeDoesNotHoldObject + nodeHoldsObject +) + +func (st nodeProcessStatus) Processed() bool { + return st != nodeNotProcessed +} + +// nodeCache tracks Policer's check progress. +type nodeCache map[uint64]bool + +func newNodeCache() nodeCache { + return make(map[uint64]bool) +} + +func (n nodeCache) set(node netmap.NodeInfo, val bool) { + n[node.Hash()] = val +} + +// submits storage node as a candidate to store the object replica in case of +// shortage. +func (n nodeCache) submitReplicaCandidate(node netmap.NodeInfo) { + n.set(node, false) +} + +// submits storage node as a current object replica holder. +func (n nodeCache) submitReplicaHolder(node netmap.NodeInfo) { + n.set(node, true) +} + +// processStatus returns current processing status of the storage node. +func (n nodeCache) processStatus(node netmap.NodeInfo) nodeProcessStatus { + switch val, ok := n[node.Hash()]; { + case !ok: + return nodeNotProcessed + case val: + return nodeHoldsObject + default: + return nodeDoesNotHoldObject + } +} + +// SubmitSuccessfulReplication marks given storage node as a current object +// replica holder. +// +// SubmitSuccessfulReplication implements replicator.TaskResult. +func (n nodeCache) SubmitSuccessfulReplication(node netmap.NodeInfo) { + n.submitReplicaHolder(node) +} diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go new file mode 100644 index 000000000..5058b026b --- /dev/null +++ b/pkg/services/policer/option.go @@ -0,0 +1,172 @@ +package policer + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/panjf2000/ants/v2" + "go.uber.org/zap" +) + +// KeySpaceIterator is the interface that allows iterating over the key space +// of local storage. +// Note that the underlying implementation might be circular: i.e. it can restart +// when the end of the key space is reached. +type KeySpaceIterator interface { + Next(context.Context, uint32) ([]objectcore.AddressWithType, error) + Rewind() +} + +// RedundantCopyCallback is a callback to pass +// the redundant local copy of the object. +type RedundantCopyCallback func(context.Context, oid.Address) + +// BuryFunc is the function to bury (i.e. inhume) an object. +type BuryFunc func(context.Context, oid.Address) error + +// Replicator is the interface to a consumer of replication tasks. +type Replicator interface { + HandleTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) +} + +// RemoteObjectHeaderFunc is the function to obtain HEAD info from a specific remote node. +type RemoteObjectHeaderFunc func(context.Context, netmapSDK.NodeInfo, oid.Address) (*objectSDK.Object, error) + +type cfg struct { + headTimeout time.Duration + + log *logger.Logger + + keySpaceIterator KeySpaceIterator + + buryFn BuryFunc + + cnrSrc container.Source + + placementBuilder placement.Builder + + remoteHeader RemoteObjectHeaderFunc + + netmapKeys netmap.AnnouncedKeys + + replicator Replicator + + cbRedundantCopy RedundantCopyCallback + + taskPool *ants.Pool + + maxCapacity int + + batchSize, cacheSize uint32 + + rebalanceFreq, evictDuration, sleepDuration time.Duration +} + +func defaultCfg() *cfg { + return &cfg{ + log: &logger.Logger{Logger: zap.L()}, + batchSize: 10, + cacheSize: 1024, // 1024 * address size = 1024 * 64 = 64 MiB + rebalanceFreq: 1 * time.Second, + sleepDuration: 1 * time.Second, + evictDuration: 30 * time.Second, + } +} + +// Option is an option for Policer constructor. +type Option func(*cfg) + +// WithHeadTimeout returns option to set Head timeout of Policer. +func WithHeadTimeout(v time.Duration) Option { + return func(c *cfg) { + c.headTimeout = v + } +} + +// WithLogger returns option to set Logger of Policer. +func WithLogger(v *logger.Logger) Option { + return func(c *cfg) { + c.log = v + } +} + +func WithKeySpaceIterator(it KeySpaceIterator) Option { + return func(c *cfg) { + c.keySpaceIterator = it + } +} + +func WithBuryFunc(f BuryFunc) Option { + return func(c *cfg) { + c.buryFn = f + } +} + +// WithContainerSource returns option to set container source of Policer. +func WithContainerSource(v container.Source) Option { + return func(c *cfg) { + c.cnrSrc = v + } +} + +// WithPlacementBuilder returns option to set object placement builder of Policer. +func WithPlacementBuilder(v placement.Builder) Option { + return func(c *cfg) { + c.placementBuilder = v + } +} + +// WithRemoteObjectHeader returns option to set object header receiver of Policer. +func WithRemoteObjectHeaderFunc(v RemoteObjectHeaderFunc) Option { + return func(c *cfg) { + c.remoteHeader = v + } +} + +// WithNetmapKeys returns option to set tool to work with announced public keys. +func WithNetmapKeys(v netmap.AnnouncedKeys) Option { + return func(c *cfg) { + c.netmapKeys = v + } +} + +// WithReplicator returns option to set object replicator of Policer. +func WithReplicator(v Replicator) Option { + return func(c *cfg) { + c.replicator = v + } +} + +// WithRedundantCopyCallback returns option to set +// callback to pass redundant local object copies +// detected by Policer. +func WithRedundantCopyCallback(cb RedundantCopyCallback) Option { + return func(c *cfg) { + c.cbRedundantCopy = cb + } +} + +// WithMaxCapacity returns option to set max capacity +// that can be set to the pool. +func WithMaxCapacity(capacity int) Option { + return func(c *cfg) { + c.maxCapacity = capacity + } +} + +// WithPool returns option to set pool for +// policy and replication operations. +func WithPool(p *ants.Pool) Option { + return func(c *cfg) { + c.taskPool = p + } +} diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index 541ab599c..a68b194d4 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -1,52 +1,40 @@ package policer import ( - "context" "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - headsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/head" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" - "github.com/panjf2000/ants/v2" "go.uber.org/zap" ) -// NodeLoader provides application load statistics. -type nodeLoader interface { - // ObjectServiceLoad returns object service load value in [0:1] range. - ObjectServiceLoad() float64 -} - type objectsInWork struct { - m sync.RWMutex + sync.RWMutex objs map[oid.Address]struct{} } func (oiw *objectsInWork) inWork(addr oid.Address) bool { - oiw.m.RLock() + oiw.RLock() _, ok := oiw.objs[addr] - oiw.m.RUnlock() + oiw.RUnlock() return ok } func (oiw *objectsInWork) remove(addr oid.Address) { - oiw.m.Lock() + oiw.Lock() delete(oiw.objs, addr) - oiw.m.Unlock() + oiw.Unlock() } -func (oiw *objectsInWork) add(addr oid.Address) { - oiw.m.Lock() +func (oiw *objectsInWork) add(addr oid.Address) bool { + oiw.Lock() + _, exists := oiw.objs[addr] oiw.objs[addr] = struct{}{} - oiw.m.Unlock() + oiw.Unlock() + return !exists } // Policer represents the utility that verifies @@ -59,53 +47,6 @@ type Policer struct { objsInWork *objectsInWork } -// Option is an option for Policer constructor. -type Option func(*cfg) - -// RedundantCopyCallback is a callback to pass -// the redundant local copy of the object. -type RedundantCopyCallback func(context.Context, oid.Address) - -type cfg struct { - headTimeout time.Duration - - log *logger.Logger - - jobQueue jobQueue - - cnrSrc container.Source - - placementBuilder placement.Builder - - remoteHeader *headsvc.RemoteHeader - - netmapKeys netmap.AnnouncedKeys - - replicator *replicator.Replicator - - cbRedundantCopy RedundantCopyCallback - - taskPool *ants.Pool - - loader nodeLoader - - maxCapacity int - - batchSize, cacheSize uint32 - - rebalanceFreq, evictDuration time.Duration -} - -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - batchSize: 10, - cacheSize: 1024, // 1024 * address size = 1024 * 64 = 64 MiB - rebalanceFreq: 1 * time.Second, - evictDuration: 30 * time.Second, - } -} - // New creates, initializes and returns Policer instance. func New(opts ...Option) *Policer { c := defaultCfg() @@ -129,91 +70,3 @@ func New(opts ...Option) *Policer { }, } } - -// WithHeadTimeout returns option to set Head timeout of Policer. -func WithHeadTimeout(v time.Duration) Option { - return func(c *cfg) { - c.headTimeout = v - } -} - -// WithLogger returns option to set Logger of Policer. -func WithLogger(v *logger.Logger) Option { - return func(c *cfg) { - c.log = v - } -} - -// WithLocalStorage returns option to set local object storage of Policer. -func WithLocalStorage(v *engine.StorageEngine) Option { - return func(c *cfg) { - c.jobQueue.localStorage = v - } -} - -// WithContainerSource returns option to set container source of Policer. -func WithContainerSource(v container.Source) Option { - return func(c *cfg) { - c.cnrSrc = v - } -} - -// WithPlacementBuilder returns option to set object placement builder of Policer. -func WithPlacementBuilder(v placement.Builder) Option { - return func(c *cfg) { - c.placementBuilder = v - } -} - -// WithRemoteHeader returns option to set object header receiver of Policer. -func WithRemoteHeader(v *headsvc.RemoteHeader) Option { - return func(c *cfg) { - c.remoteHeader = v - } -} - -// WithNetmapKeys returns option to set tool to work with announced public keys. -func WithNetmapKeys(v netmap.AnnouncedKeys) Option { - return func(c *cfg) { - c.netmapKeys = v - } -} - -// WithReplicator returns option to set object replicator of Policer. -func WithReplicator(v *replicator.Replicator) Option { - return func(c *cfg) { - c.replicator = v - } -} - -// WithRedundantCopyCallback returns option to set -// callback to pass redundant local object copies -// detected by Policer. -func WithRedundantCopyCallback(cb RedundantCopyCallback) Option { - return func(c *cfg) { - c.cbRedundantCopy = cb - } -} - -// WithMaxCapacity returns option to set max capacity -// that can be set to the pool. -func WithMaxCapacity(capacity int) Option { - return func(c *cfg) { - c.maxCapacity = capacity - } -} - -// WithPool returns option to set pool for -// policy and replication operations. -func WithPool(p *ants.Pool) Option { - return func(c *cfg) { - c.taskPool = p - } -} - -// WithNodeLoader returns option to set FrostFS node load source. -func WithNodeLoader(l nodeLoader) Option { - return func(c *cfg) { - c.loader = l - } -} diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go new file mode 100644 index 000000000..c0aeac515 --- /dev/null +++ b/pkg/services/policer/policer_test.go @@ -0,0 +1,378 @@ +package policer + +import ( + "bytes" + "context" + "errors" + "sort" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/panjf2000/ants/v2" + "github.com/stretchr/testify/require" +) + +func TestBuryObjectWithoutContainer(t *testing.T) { + // Key space + addr := oidtest.Address() + objs := []objectcore.AddressWithType{ + { + Address: addr, + Type: objectSDK.TypeRegular, + }, + } + + // Container source and bury function + buryCh := make(chan oid.Address) + containerSrc := func(id cid.ID) (*container.Container, error) { + return nil, apistatus.ContainerNotFound{} + } + buryFn := func(ctx context.Context, a oid.Address) error { + buryCh <- a + return nil + } + + // Task pool + pool, err := ants.NewPool(4) + require.NoError(t, err) + + // Policer instance + p := New( + WithKeySpaceIterator(&sliceKeySpaceIterator{objs: objs}), + WithContainerSource(containerSrcFunc(containerSrc)), + WithBuryFunc(buryFn), + WithPool(pool), + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Run(ctx) + + require.Equal(t, addr, <-buryCh) +} + +func TestProcessObject(t *testing.T) { + // Notes: + // - nodes are referred to by their index throughout, which is embedded in the public key + // - node with index 0 always refers to the local node, so there's no need to add it to objHolders + // - policy is used only to match the number of replicas for each index in the placement + tests := []struct { + desc string + objType objectSDK.Type + nodeCount int + policy string + placement [][]int + objHolders []int + maintenanceNodes []int + wantRemoveRedundant bool + wantReplicateTo []int + }{ + { + desc: "1 copy already held by local node", + nodeCount: 1, + policy: `REP 1`, + placement: [][]int{{0}}, + }, + { + desc: "1 copy already held by the remote node", + nodeCount: 2, + policy: `REP 1`, + placement: [][]int{{1}}, + objHolders: []int{1}, + wantRemoveRedundant: true, + }, + { + desc: "1 copy not yet held by the remote node", + nodeCount: 2, + policy: `REP 1`, + placement: [][]int{{1}}, + wantReplicateTo: []int{1}, + }, + { + desc: "2 copies already held by local and remote node", + nodeCount: 2, + policy: `REP 2`, + placement: [][]int{{0, 1}}, + objHolders: []int{1}, + }, + { + desc: "2 copies but not held by remote node", + nodeCount: 2, + policy: `REP 2`, + placement: [][]int{{0, 1}}, + wantReplicateTo: []int{1}, + }, + { + desc: "multiple vectors already held by remote node", + nodeCount: 2, + policy: `REP 2 REP 2`, + placement: [][]int{{0, 1}, {0, 1}}, + objHolders: []int{1}, + }, + { + desc: "multiple vectors not yet held by remote node", + nodeCount: 2, + policy: `REP 2 REP 2`, + placement: [][]int{{0, 1}, {0, 1}}, + wantReplicateTo: []int{1, 1}, // is this actually good? + }, + { + desc: "lock object must be replicated to all nodes", + objType: objectSDK.TypeLock, + nodeCount: 3, + policy: `REP 1`, + placement: [][]int{{0, 1, 2}}, + wantReplicateTo: []int{1, 2}, + }, + { + desc: "preserve local copy when maintenance nodes exist", + nodeCount: 3, + policy: `REP 2`, + placement: [][]int{{1, 2}}, + objHolders: []int{1}, + maintenanceNodes: []int{2}, + }, + } + + for i := range tests { + ti := tests[i] + t.Run(ti.desc, func(t *testing.T) { + addr := oidtest.Address() + + // Netmap, placement policy and placement builder + nodes := make([]netmap.NodeInfo, ti.nodeCount) + for i := range nodes { + nodes[i].SetPublicKey([]byte{byte(i)}) + } + for _, i := range ti.maintenanceNodes { + nodes[i].SetMaintenance() + } + + var policy netmap.PlacementPolicy + require.NoError(t, policy.DecodeString(ti.policy)) + + placementVectors := make([][]netmap.NodeInfo, len(ti.placement)) + for i, pv := range ti.placement { + for _, nj := range pv { + placementVectors[i] = append(placementVectors[i], nodes[nj]) + } + } + placementBuilder := func(cnr cid.ID, obj *oid.ID, p netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { + if cnr.Equals(addr.Container()) && obj != nil && obj.Equals(addr.Object()) { + return placementVectors, nil + } + t.Errorf("unexpected placement build: cid=%v oid=%v", cnr, obj) + return nil, errors.New("unexpected placement build") + } + + // Object remote header + headFn := func(_ context.Context, ni netmap.NodeInfo, a oid.Address) (*objectSDK.Object, error) { + index := int(ni.PublicKey()[0]) + if a != addr || index < 1 || index >= ti.nodeCount { + t.Errorf("unexpected remote object head: node=%+v addr=%v", ni, a) + return nil, errors.New("unexpected object head") + } + for _, i := range ti.objHolders { + if index == i { + return nil, nil + } + } + return nil, apistatus.ObjectNotFound{} + } + + // Container source + cnr := &container.Container{} + cnr.Value.Init() + cnr.Value.SetPlacementPolicy(policy) + containerSrc := func(id cid.ID) (*container.Container, error) { + if id.Equals(addr.Container()) { + return cnr, nil + } + t.Errorf("unexpected container requested: got=%v, want=%v", id, addr.Container()) + return nil, apistatus.ContainerNotFound{} + } + buryFn := func(ctx context.Context, a oid.Address) error { + t.Errorf("unexpected object buried: %v", a) + return nil + } + + // Policer instance + var gotRemoveRedundant bool + var gotReplicateTo []int + + p := New( + WithContainerSource(containerSrcFunc(containerSrc)), + WithPlacementBuilder(placementBuilderFunc(placementBuilder)), + WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { + return bytes.Equal(k, nodes[0].PublicKey()) + })), + WithRemoteObjectHeaderFunc(headFn), + WithBuryFunc(buryFn), + WithRedundantCopyCallback(func(_ context.Context, a oid.Address) { + require.True(t, a.Equals(addr), "unexpected redundant copy callback: a=%v", a) + gotRemoveRedundant = true + }), + WithReplicator(replicatorFunc(func(_ context.Context, task replicator.Task, res replicator.TaskResult) { + require.True(t, task.Addr.Equals(addr), "unexpected replicator task: %+v", task) + for _, node := range task.Nodes { + gotReplicateTo = append(gotReplicateTo, int(node.PublicKey()[0])) + } + })), + ) + + addrWithType := objectcore.AddressWithType{ + Address: addr, + Type: ti.objType, + } + + p.processObject(context.Background(), addrWithType) + sort.Ints(gotReplicateTo) + + require.Equal(t, ti.wantRemoveRedundant, gotRemoveRedundant) + require.Equal(t, ti.wantReplicateTo, gotReplicateTo) + }) + } +} + +func TestIteratorContract(t *testing.T) { + addr := oidtest.Address() + objs := []objectcore.AddressWithType{{ + Address: addr, + Type: objectSDK.TypeRegular, + }} + + containerSrc := func(id cid.ID) (*container.Container, error) { + return nil, apistatus.ContainerNotFound{} + } + buryFn := func(ctx context.Context, a oid.Address) error { + return nil + } + + pool, err := ants.NewPool(4) + require.NoError(t, err) + + it := &predefinedIterator{ + scenario: []nextResult{ + {objs, nil}, + {nil, errors.New("opaque")}, + {nil, engine.ErrEndOfListing}, + {nil, engine.ErrEndOfListing}, + {nil, errors.New("opaque")}, + {objs, engine.ErrEndOfListing}, + }, + finishCh: make(chan struct{}), + } + + p := New( + WithKeySpaceIterator(it), + WithContainerSource(containerSrcFunc(containerSrc)), + WithBuryFunc(buryFn), + WithPool(pool), + func(c *cfg) { + c.sleepDuration = time.Millisecond + }, + ) + + ctx, cancel := context.WithCancel(context.Background()) + go p.Run(ctx) + + <-it.finishCh + cancel() + require.Equal(t, []string{ + "Next", + "Next", + "Next", + "Rewind", + "Next", + "Rewind", + "Next", + "Next", + "Rewind", + }, it.calls) +} + +type nextResult struct { + objs []objectcore.AddressWithType + err error +} + +type predefinedIterator struct { + scenario []nextResult + finishCh chan struct{} + pos int + calls []string +} + +func (it *predefinedIterator) Next(ctx context.Context, size uint32) ([]objectcore.AddressWithType, error) { + if it.pos == len(it.scenario) { + close(it.finishCh) + <-ctx.Done() + return nil, nil + } + + res := it.scenario[it.pos] + it.pos += 1 + it.calls = append(it.calls, "Next") + return res.objs, res.err +} + +func (it *predefinedIterator) Rewind() { + it.calls = append(it.calls, "Rewind") +} + +// sliceKeySpaceIterator is a KeySpaceIterator backed by a slice. +type sliceKeySpaceIterator struct { + objs []objectcore.AddressWithType + cur int +} + +func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectcore.AddressWithType, error) { + if it.cur >= len(it.objs) { + return nil, engine.ErrEndOfListing + } + end := it.cur + int(size) + if end > len(it.objs) { + end = len(it.objs) + } + ret := it.objs[it.cur:end] + it.cur = end + return ret, nil +} + +func (it *sliceKeySpaceIterator) Rewind() { + it.cur = 0 +} + +// containerSrcFunc is a container.Source backed by a function. +type containerSrcFunc func(cid.ID) (*container.Container, error) + +func (f containerSrcFunc) Get(id cid.ID) (*container.Container, error) { return f(id) } + +// placementBuilderFunc is a placement.Builder backed by a function +type placementBuilderFunc func(cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) + +func (f placementBuilderFunc) BuildPlacement(c cid.ID, o *oid.ID, p netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { + return f(c, o, p) +} + +// announcedKeysFunc is a netmap.AnnouncedKeys backed by a function. +type announcedKeysFunc func([]byte) bool + +func (f announcedKeysFunc) IsLocalKey(k []byte) bool { return f(k) } + +// replicatorFunc is a Replicator backed by a function. +type replicatorFunc func(context.Context, replicator.Task, replicator.TaskResult) + +func (f replicatorFunc) HandleTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) { + f(ctx, task, res) +} diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 4a40f00ba..1f61c69f4 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -6,27 +6,16 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "go.uber.org/zap" ) func (p *Policer) Run(ctx context.Context) { - defer func() { - p.log.Info(logs.PolicerRoutineStopped) - }() - - go p.poolCapacityWorker(ctx) p.shardPolicyWorker(ctx) + p.log.Info(logs.PolicerRoutineStopped) } func (p *Policer) shardPolicyWorker(ctx context.Context) { - var ( - addrs []objectcore.AddressWithType - cursor *engine.Cursor - err error - ) - for { select { case <-ctx.Done(): @@ -34,10 +23,11 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { default: } - addrs, cursor, err = p.jobQueue.Select(cursor, p.batchSize) + addrs, err := p.keySpaceIterator.Next(ctx, p.batchSize) if err != nil { if errors.Is(err, engine.ErrEndOfListing) { - time.Sleep(time.Second) // finished whole cycle, sleep a bit + p.keySpaceIterator.Rewind() + time.Sleep(p.sleepDuration) // finished whole cycle, sleep a bit continue } p.log.Warn(logs.PolicerFailureAtObjectSelectForReplication, zap.Error(err)) @@ -55,18 +45,17 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { continue } - err = p.taskPool.Submit(func() { + err := p.taskPool.Submit(func() { v, ok := p.cache.Get(addr.Address) if ok && time.Since(v) < p.evictDuration { return } - p.objsInWork.add(addr.Address) - - p.processObject(ctx, addr) - - p.cache.Add(addr.Address, time.Now()) - p.objsInWork.remove(addr.Address) + if p.objsInWork.add(addr.Address) { + p.processObject(ctx, addr) + p.cache.Add(addr.Address, time.Now()) + p.objsInWork.remove(addr.Address) + } }) if err != nil { p.log.Warn(logs.PolicerPoolSubmission, zap.Error(err)) @@ -75,27 +64,3 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { } } } - -func (p *Policer) poolCapacityWorker(ctx context.Context) { - ticker := time.NewTicker(p.rebalanceFreq) - for { - select { - case <-ctx.Done(): - ticker.Stop() - return - case <-ticker.C: - frostfsSysLoad := p.loader.ObjectServiceLoad() - newCapacity := int((1.0 - frostfsSysLoad) * float64(p.maxCapacity)) - if newCapacity == 0 { - newCapacity++ - } - - if p.taskPool.Cap() != newCapacity { - p.taskPool.Tune(newCapacity) - p.log.Debug(logs.PolicerTuneReplicationCapacity, - zap.Float64("system_load", frostfsSysLoad), - zap.Int("new_capacity", newCapacity)) - } - } - } -} diff --git a/pkg/services/policer/queue.go b/pkg/services/policer/queue.go deleted file mode 100644 index b8af44049..000000000 --- a/pkg/services/policer/queue.go +++ /dev/null @@ -1,25 +0,0 @@ -package policer - -import ( - "fmt" - - objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" -) - -type jobQueue struct { - localStorage *engine.StorageEngine -} - -func (q *jobQueue) Select(cursor *engine.Cursor, count uint32) ([]objectcore.AddressWithType, *engine.Cursor, error) { - var prm engine.ListWithCursorPrm - prm.WithCursor(cursor) - prm.WithCount(count) - - res, err := q.localStorage.ListWithCursor(prm) - if err != nil { - return nil, nil, fmt.Errorf("cannot list objects in engine: %w", err) - } - - return res.AddressList(), res.Cursor(), nil -} diff --git a/pkg/services/replicator/metrics.go b/pkg/services/replicator/metrics.go new file mode 100644 index 000000000..3fc062926 --- /dev/null +++ b/pkg/services/replicator/metrics.go @@ -0,0 +1,8 @@ +package replicator + +type MetricsRegister interface { + IncInFlightRequest() + DecInFlightRequest() + IncProcessedObjects() + AddPayloadSize(size int64) +} diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 46e0c9468..16bcec9c5 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -6,7 +6,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -20,18 +23,27 @@ type TaskResult interface { // HandleTask executes replication task inside invoking goroutine. // Passes all the nodes that accepted the replication to the TaskResult. func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) { + p.metrics.IncInFlightRequest() + defer p.metrics.DecInFlightRequest() defer func() { p.log.Debug(logs.ReplicatorFinishWork, - zap.Uint32("amount of unfinished replicas", task.quantity), + zap.Uint32("amount of unfinished replicas", task.NumCopies), ) }() - if task.obj == nil { + ctx, span := tracing.StartSpanFromContext(ctx, "Replicator.HandleTask", + trace.WithAttributes( + attribute.Stringer("address", task.Addr), + attribute.Int64("number_of_copies", int64(task.NumCopies)), + )) + defer span.End() + + if task.Obj == nil { var err error - task.obj, err = engine.Get(ctx, p.localStorage, task.addr) + task.Obj, err = engine.Get(ctx, p.localStorage, task.Addr) if err != nil { p.log.Error(logs.ReplicatorCouldNotGetObjectFromLocalStorage, - zap.Stringer("object", task.addr), + zap.Stringer("object", task.Addr), zap.Error(err)) return @@ -39,9 +51,9 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) } prm := new(putsvc.RemotePutPrm). - WithObject(task.obj) + WithObject(task.Obj) - for i := 0; task.quantity > 0 && i < len(task.nodes); i++ { + for i := 0; task.NumCopies > 0 && i < len(task.Nodes); i++ { select { case <-ctx.Done(): return @@ -49,13 +61,13 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) } log := p.log.With( - zap.String("node", netmap.StringifyPublicKey(task.nodes[i])), - zap.Stringer("object", task.addr), + zap.String("node", netmap.StringifyPublicKey(task.Nodes[i])), + zap.Stringer("object", task.Addr), ) callCtx, cancel := context.WithTimeout(ctx, p.putTimeout) - err := p.remoteSender.PutObject(callCtx, prm.WithNodeInfo(task.nodes[i])) + err := p.remoteSender.PutObject(callCtx, prm.WithNodeInfo(task.Nodes[i])) cancel() @@ -66,9 +78,12 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) } else { log.Debug(logs.ReplicatorObjectSuccessfullyReplicated) - task.quantity-- + task.NumCopies-- - res.SubmitSuccessfulReplication(task.nodes[i]) + res.SubmitSuccessfulReplication(task.Nodes[i]) + + p.metrics.IncProcessedObjects() + p.metrics.AddPayloadSize(int64(task.Obj.PayloadSize())) } } } diff --git a/pkg/services/replicator/replicator.go b/pkg/services/replicator/replicator.go index 493982100..bb817cb32 100644 --- a/pkg/services/replicator/replicator.go +++ b/pkg/services/replicator/replicator.go @@ -26,6 +26,8 @@ type cfg struct { remoteSender *putsvc.RemoteSender localStorage *engine.StorageEngine + + metrics MetricsRegister } func defaultCfg() *cfg { @@ -74,3 +76,9 @@ func WithLocalStorage(v *engine.StorageEngine) Option { c.localStorage = v } } + +func WithMetrics(v MetricsRegister) Option { + return func(c *cfg) { + c.metrics = v + } +} diff --git a/pkg/services/replicator/task.go b/pkg/services/replicator/task.go index ec1b55788..d2b5b2506 100644 --- a/pkg/services/replicator/task.go +++ b/pkg/services/replicator/task.go @@ -8,31 +8,12 @@ import ( // Task represents group of Replicator task parameters. type Task struct { - quantity uint32 - - addr oid.Address - - obj *objectSDK.Object - - nodes []netmap.NodeInfo -} - -// SetCopiesNumber sets number of copies to replicate. -func (t *Task) SetCopiesNumber(v uint32) { - t.quantity = v -} - -// SetObjectAddress sets address of local object. -func (t *Task) SetObjectAddress(v oid.Address) { - t.addr = v -} - -// SetObject sets object to avoid fetching it from the local storage. -func (t *Task) SetObject(obj *objectSDK.Object) { - t.obj = obj -} - -// SetNodes sets a list of potential object holders. -func (t *Task) SetNodes(v []netmap.NodeInfo) { - t.nodes = v + // NumCopies is the number of copies to replicate. + NumCopies uint32 + // Addr is the address of the local object. + Addr oid.Address + // Obj is the object to avoid fetching it from the local storage. + Obj *objectSDK.Object + // Nodes is a list of potential object holders. + Nodes []netmap.NodeInfo } diff --git a/pkg/services/session/executor.go b/pkg/services/session/executor.go index 5ad1d6518..76c220fab 100644 --- a/pkg/services/session/executor.go +++ b/pkg/services/session/executor.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -17,14 +18,17 @@ type ServiceExecutor interface { type executorSvc struct { exec ServiceExecutor + respSvc *response.Service + log *logger.Logger } // NewExecutionService wraps ServiceExecutor and returns Session Service interface. -func NewExecutionService(exec ServiceExecutor, l *logger.Logger) Server { +func NewExecutionService(exec ServiceExecutor, respSvc *response.Service, l *logger.Logger) Server { return &executorSvc{ - exec: exec, - log: l, + exec: exec, + log: l, + respSvc: respSvc, } } @@ -42,5 +46,6 @@ func (s *executorSvc) Create(ctx context.Context, req *session.CreateRequest) (* resp := new(session.CreateResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/session/response.go b/pkg/services/session/response.go deleted file mode 100644 index cbf93fb1f..000000000 --- a/pkg/services/session/response.go +++ /dev/null @@ -1,37 +0,0 @@ -package session - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns session service instance that passes internal service -// call to response service. -func NewResponseService(ssSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: ssSvc, - } -} - -func (s *responseService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Create(ctx, req.(*session.CreateRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*session.CreateResponse), nil -} diff --git a/pkg/services/session/sign.go b/pkg/services/session/sign.go index 1156dc538..ffce0621e 100644 --- a/pkg/services/session/sign.go +++ b/pkg/services/session/sign.go @@ -22,17 +22,10 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { } func (s *signService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Create(ctx, req.(*session.CreateRequest)) - }, - func() util.ResponseMessage { - return new(session.CreateResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(session.CreateResponse) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } - - return resp.(*session.CreateResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Create(ctx, req)) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/session/storage/temporary/storage.go b/pkg/services/session/storage/temporary/storage.go index 370499e06..ee93dee71 100644 --- a/pkg/services/session/storage/temporary/storage.go +++ b/pkg/services/session/storage/temporary/storage.go @@ -18,7 +18,7 @@ type key struct { // expiring (removing) session tokens. // Must be created only via calling NewTokenStore. type TokenStore struct { - mtx *sync.RWMutex + mtx sync.RWMutex tokens map[key]*storage.PrivateToken } @@ -28,7 +28,6 @@ type TokenStore struct { // The elements of the instance are stored in the map. func NewTokenStore() *TokenStore { return &TokenStore{ - mtx: new(sync.RWMutex), tokens: make(map[key]*storage.PrivateToken), } } diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 56b97e687..f50aa0b0d 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -4,12 +4,12 @@ import ( "context" "errors" "fmt" - "strings" "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "github.com/hashicorp/golang-lru/v2/simplelru" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" @@ -27,7 +27,7 @@ type cacheItem struct { } const ( - defaultClientCacheSize = 10 + defaultClientCacheSize = 32 defaultClientConnectTimeout = time.Second * 2 defaultReconnectInterval = time.Second * 15 ) @@ -90,15 +90,16 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, opts := []grpc.DialOption{ grpc.WithBlock(), grpc.WithChainUnaryInterceptor( - tracing.NewGRPCUnaryClientInteceptor(), + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( - tracing.NewGRPCStreamClientInterceptor(), + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), ), } - // FIXME(@fyrchik): ugly hack #1322 - if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") { + if !netAddr.IsTLSEnabled() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index dc4ce29aa..88a5b5e06 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -3,6 +3,8 @@ package tree import ( "context" "errors" + "path" + "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -119,6 +121,65 @@ func TestGetSubTree(t *testing.T) { }) } +func TestGetSubTreeOrderAsc(t *testing.T) { + d := pilorama.CIDDescriptor{CID: cidtest.ID(), Size: 1} + treeID := "sometree" + p := pilorama.NewMemoryForest() + + tree := []struct { + path []string + id uint64 + }{ + {path: []string{"dir1"}}, + {path: []string{"dir2"}}, + {path: []string{"dir1", "sub1"}}, + {path: []string{"dir2", "sub1"}}, + {path: []string{"dir2", "sub2"}}, + {path: []string{"dir2", "sub1", "subsub1"}}, + } + + for i := range tree { + path := tree[i].path + meta := []pilorama.KeyValue{ + {Key: pilorama.AttributeFilename, Value: []byte(path[len(path)-1])}} + + lm, err := p.TreeAddByPath(context.Background(), d, treeID, pilorama.AttributeFilename, path[:len(path)-1], meta) + require.NoError(t, err) + require.Equal(t, 1, len(lm)) + tree[i].id = lm[0].Child + } + + acc := subTreeAcc{errIndex: -1} + err := getSubTree(context.Background(), &acc, d.CID, &GetSubTreeRequest_Body{ + TreeId: treeID, + OrderBy: &GetSubTreeRequest_Body_Order{ + Direction: GetSubTreeRequest_Body_Order_Asc, + }, + }, p) + require.NoError(t, err) + // GetSubTree must return child only after is has returned the parent. + require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) + + paths := make([]string, 0, len(acc.seen)) + for i := range acc.seen { + if i == 0 { + continue + } + found := false + for j := range tree { + if acc.seen[i].Body.NodeId == tree[j].id { + found = true + paths = append(paths, path.Join(tree[j].path...)) + } + } + require.True(t, found, "unknown node") + } + + require.True(t, sort.SliceIsSorted(paths, func(i, j int) bool { + return paths[i] < paths[j] + })) +} + var ( errSubTreeSend = errors.New("send finished with error") errSubTreeSendAfterError = errors.New("send was invoked after an error occurred") diff --git a/pkg/services/tree/metrics.go b/pkg/services/tree/metrics.go new file mode 100644 index 000000000..0f0e4ee57 --- /dev/null +++ b/pkg/services/tree/metrics.go @@ -0,0 +1,15 @@ +package tree + +import "time" + +type MetricsRegister interface { + AddReplicateTaskDuration(time.Duration, bool) + AddReplicateWaitDuration(time.Duration, bool) + AddSyncDuration(time.Duration, bool) +} + +type defaultMetricsRegister struct{} + +func (defaultMetricsRegister) AddReplicateTaskDuration(time.Duration, bool) {} +func (defaultMetricsRegister) AddReplicateWaitDuration(time.Duration, bool) {} +func (defaultMetricsRegister) AddSyncDuration(time.Duration, bool) {} diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index d60bc14c5..a6e23c625 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -33,6 +33,9 @@ type cfg struct { replicatorWorkerCount int replicatorTimeout time.Duration containerCacheSize int + authorizedKeys [][]byte + + metrics MetricsRegister } // Option represents configuration option for a tree service. @@ -116,3 +119,20 @@ func WithReplicationTimeout(t time.Duration) Option { } } } + +func WithMetrics(v MetricsRegister) Option { + return func(c *cfg) { + c.metrics = v + } +} + +// WithAuthorizedKeys returns option to add list of public +// keys that have rights to use Tree service. +func WithAuthorizedKeys(keys keys.PublicKeys) Option { + return func(c *cfg) { + c.authorizedKeys = nil + for _, key := range keys { + c.authorizedKeys = append(c.authorizedKeys, key.Bytes()) + } + } +} diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 3de71b554..0afd3439a 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -5,8 +5,8 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 60d0eff50..0ca30273e 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -8,9 +8,9 @@ import ( "fmt" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" @@ -75,6 +75,7 @@ func (s *Service) replicationWorker(ctx context.Context) { attribute.String("public_key", hex.EncodeToString(task.n.PublicKey())), ), ) + start := time.Now() var lastErr error var lastAddr string @@ -113,6 +114,9 @@ func (s *Service) replicationWorker(ctx context.Context) { zap.String("address", lastAddr), zap.String("key", hex.EncodeToString(task.n.PublicKey()))) } + s.metrics.AddReplicateTaskDuration(time.Since(start), false) + } else { + s.metrics.AddReplicateTaskDuration(time.Since(start), true) } span.End() } @@ -137,6 +141,7 @@ func (s *Service) replicateLoop(ctx context.Context) { case <-ctx.Done(): return case op := <-s.replicateCh: + start := time.Now() err := s.replicate(op) if err != nil { s.log.Error(logs.TreeErrorDuringReplication, @@ -144,6 +149,7 @@ func (s *Service) replicateLoop(ctx context.Context) { zap.Stringer("cid", op.cid), zap.String("treeID", op.treeID)) } + s.metrics.AddReplicateWaitDuration(time.Since(start), err == nil) } } } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 546b7a207..57767f87e 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -5,7 +5,9 @@ import ( "context" "errors" "fmt" + "sort" "sync" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -31,6 +33,8 @@ type Service struct { syncChan chan struct{} syncPool *ants.Pool + initialSyncDone atomic.Bool + // cnrMap contains existing (used) container IDs. cnrMap map[cidSDK.ID]struct{} // cnrMapMtx protects cnrMap @@ -46,6 +50,7 @@ func New(opts ...Option) *Service { s.replicatorChannelCapacity = defaultReplicatorCapacity s.replicatorWorkerCount = defaultReplicatorWorkerCount s.replicatorTimeout = defaultReplicatorSendTimeout + s.metrics = defaultMetricsRegister{} for i := range opts { opts[i](&s.cfg) @@ -89,6 +94,10 @@ func (s *Service) Shutdown() { } func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -137,6 +146,10 @@ func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error } func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByPathResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -197,6 +210,10 @@ func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByP } func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -246,6 +263,10 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon // Move applies client operation to the specified tree and pushes in queue // for replication on other nodes. func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -294,6 +315,10 @@ func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, er } func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) (*GetNodeByPathResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -370,6 +395,10 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) } func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error { + if !s.initialSyncDone.Load() { + return ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -412,7 +441,15 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // recursive implementation is not suitable here, so we maintain explicit stack. - stack := [][]uint64{{b.GetRootId()}} + m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId()) + if err != nil { + return err + } + stack := [][]pilorama.NodeInfo{{{ + ID: b.GetRootId(), + Meta: m, + ParentID: p, + }}} for { if len(stack) == 0 { @@ -422,19 +459,15 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD continue } - nodeID := stack[len(stack)-1][0] + node := stack[len(stack)-1][0] stack[len(stack)-1] = stack[len(stack)-1][1:] - m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), nodeID) - if err != nil { - return err - } err = srv.Send(&GetSubTreeResponse{ Body: &GetSubTreeResponse_Body{ - NodeId: nodeID, - ParentId: p, - Timestamp: m.Time, - Meta: metaToProto(m.Items), + NodeId: node.ID, + ParentId: node.ParentID, + Timestamp: node.Meta.Time, + Meta: metaToProto(node.Meta.Items), }, }) if err != nil { @@ -442,7 +475,11 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD } if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() { - children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), nodeID) + children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), node.ID) + if err != nil { + return err + } + children, err = sortByFilename(children, b.GetOrderBy().GetDirection()) if err != nil { return err } @@ -454,6 +491,24 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD return nil } +func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Direction) ([]pilorama.NodeInfo, error) { + switch d { + case GetSubTreeRequest_Body_Order_None: + return nodes, nil + case GetSubTreeRequest_Body_Order_Asc: + if len(nodes) == 0 { + return nodes, nil + } + less := func(i, j int) bool { + return bytes.Compare(nodes[i].Meta.GetAttr(pilorama.AttributeFilename), nodes[j].Meta.GetAttr(pilorama.AttributeFilename)) < 0 + } + sort.Slice(nodes, less) + return nodes, nil + default: + return nil, fmt.Errorf("unsupported order direction: %s", d.String()) + } +} + // Apply locally applies operation from the remote node to the tree. func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) { err := verifyMessage(req) @@ -499,6 +554,10 @@ func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, e } func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) error { + if !s.initialSyncDone.Load() { + return ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -531,9 +590,13 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) } h := b.GetHeight() + lastHeight, err := s.forest.TreeHeight(srv.Context(), cid, b.GetTreeId()) + if err != nil { + return err + } for { lm, err := s.forest.TreeGetOpLog(srv.Context(), cid, b.GetTreeId(), h) - if err != nil || lm.Time == 0 { + if err != nil || lm.Time == 0 || lastHeight < lm.Time { return err } @@ -555,6 +618,10 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) } func (s *Service) TreeList(ctx context.Context, req *TreeListRequest) (*TreeListResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + var cid cidSDK.ID err := cid.Decode(req.GetBody().GetContainerId()) @@ -638,5 +705,9 @@ func (s *Service) getContainerInfo(cid cidSDK.ID, pub []byte) ([]netmapSDK.NodeI } func (s *Service) Healthcheck(context.Context, *HealthcheckRequest) (*HealthcheckResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + return new(HealthcheckResponse), nil } diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index 08664a6d0..63f3e714a 100644 Binary files a/pkg/services/tree/service.pb.go and b/pkg/services/tree/service.pb.go differ diff --git a/pkg/services/tree/service.proto b/pkg/services/tree/service.proto index 182d8adb2..ec63d88ec 100644 --- a/pkg/services/tree/service.proto +++ b/pkg/services/tree/service.proto @@ -238,6 +238,13 @@ message GetNodeByPathResponse { message GetSubTreeRequest { message Body { + message Order { + enum Direction { + None = 0; + Asc = 1; + } + Direction direction = 1; + } // Container ID in V2 format. bytes container_id = 1; // The name of the tree. @@ -249,6 +256,8 @@ message GetSubTreeRequest { uint32 depth = 4; // Bearer token in V2 format. bytes bearer_token = 5; + // Result ordering. + Order order_by = 6; } // Request body. diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index 42b7ba3fc..b272e4389 100644 Binary files a/pkg/services/tree/service_frostfs.pb.go and b/pkg/services/tree/service_frostfs.pb.go differ diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index fa259e804..2c0828951 100644 Binary files a/pkg/services/tree/service_grpc.pb.go and b/pkg/services/tree/service_grpc.pb.go differ diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 439912969..7a466955c 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -52,6 +52,11 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op return err } + isAuthorized, err := s.isAuthorized(req, op) + if isAuthorized || err != nil { + return err + } + cnr, err := s.cnrSource.Get(cid) if err != nil { return fmt.Errorf("can't get container %s: %w", cid, err) @@ -59,18 +64,9 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op eaclOp := eACLOp(op) - var bt *bearer.Token - if len(rawBearer) > 0 { - bt = new(bearer.Token) - if err = bt.Unmarshal(rawBearer); err != nil { - return eACLErr(eaclOp, fmt.Errorf("invalid bearer token: %w", err)) - } - if !bt.AssertContainer(cid) { - return eACLErr(eaclOp, errBearerWrongContainer) - } - if !bt.VerifySignature() { - return eACLErr(eaclOp, errBearerSignature) - } + bt, err := parseBearer(rawBearer, cid, eaclOp) + if err != nil { + return err } role, err := roleFromReq(cnr, req, bt) @@ -88,7 +84,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op return nil } - var tableFromBearer bool + var useBearer bool if len(rawBearer) != 0 { if !basicACL.AllowedBearerRules(op) { s.log.Debug(logs.TreeBearerPresentedButNotAllowedByACL, @@ -96,38 +92,71 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op zap.String("op", op.String()), ) } else { - tableFromBearer = true + useBearer = true } } var tb eacl.Table signer := req.GetSignature().GetKey() - if tableFromBearer { - if bt.Impersonate() { - tbCore, err := s.eaclSource.GetEACL(cid) - if err != nil { - return handleGetEACLError(err) - } - tb = *tbCore.Value - signer = bt.SigningKeyBytes() - } else { - if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { - return eACLErr(eaclOp, errBearerWrongOwner) - } - tb = bt.EACLTable() + if useBearer && !bt.Impersonate() { + if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { + return eACLErr(eaclOp, errBearerWrongOwner) } + tb = bt.EACLTable() } else { tbCore, err := s.eaclSource.GetEACL(cid) if err != nil { return handleGetEACLError(err) } - tb = *tbCore.Value + + if useBearer && bt.Impersonate() { + signer = bt.SigningKeyBytes() + } } return checkEACL(tb, signer, eACLRole(role), eaclOp) } +// Returns true iff the operation is read-only and request was signed +// with one of the authorized keys. +func (s *Service) isAuthorized(req message, op acl.Op) (bool, error) { + if op != acl.OpObjectGet { + return false, nil + } + + sign := req.GetSignature() + if sign == nil { + return false, errors.New("missing signature") + } + + key := sign.GetKey() + for i := range s.authorizedKeys { + if bytes.Equal(s.authorizedKeys[i], key) { + return true, nil + } + } + return false, nil +} + +func parseBearer(rawBearer []byte, cid cidSDK.ID, eaclOp eacl.Operation) (*bearer.Token, error) { + if len(rawBearer) == 0 { + return nil, nil + } + + bt := new(bearer.Token) + if err := bt.Unmarshal(rawBearer); err != nil { + return nil, eACLErr(eaclOp, fmt.Errorf("invalid bearer token: %w", err)) + } + if !bt.AssertContainer(cid) { + return nil, eACLErr(eaclOp, errBearerWrongContainer) + } + if !bt.VerifySignature() { + return nil, eACLErr(eaclOp, errBearerSignature) + } + return bt, nil +} + func handleGetEACLError(err error) error { if client.IsErrEACLNotFound(err) { return nil @@ -144,7 +173,7 @@ func verifyMessage(m message) error { sig := m.GetSignature() - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetKey(sig.GetKey()) sigV2.SetSign(sig.GetSign()) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index ed2455194..e44e8dbbf 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -9,12 +9,15 @@ import ( "math" "math/rand" "sync" + "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + tracing_grpc "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" @@ -296,10 +299,12 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, cc, err := grpc.DialContext(egCtx, a.URIAddr(), grpc.WithChainUnaryInterceptor( - tracing.NewGRPCUnaryClientInteceptor(), + metrics.NewUnaryClientInterceptor(), + tracing_grpc.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( - tracing.NewGRPCStreamClientInterceptor(), + metrics.NewStreamClientInterceptor(), + tracing_grpc.NewStreamClientInterceptor(), ), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { @@ -376,11 +381,14 @@ func (s *Service) syncLoop(ctx context.Context) { ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.sync") s.log.Debug(logs.TreeSyncingTrees) + start := time.Now() + cnrs, err := s.cfg.cnrSource.List() if err != nil { s.log.Error(logs.TreeCouldNotFetchContainers, zap.Error(err)) + s.metrics.AddSyncDuration(time.Since(start), false) span.End() - continue + break } newMap, cnrsToSync := s.containersToSync(cnrs) @@ -390,8 +398,11 @@ func (s *Service) syncLoop(ctx context.Context) { s.removeContainers(ctx, newMap) s.log.Debug(logs.TreeTreesHaveBeenSynchronized) + + s.metrics.AddSyncDuration(time.Since(start), true) span.End() } + s.initialSyncDone.Store(true) } } diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go index 45d889177..b4d6981ef 100644 Binary files a/pkg/services/tree/types.pb.go and b/pkg/services/tree/types.pb.go differ diff --git a/pkg/services/util/response/client_stream.go b/pkg/services/util/response/client_stream.go deleted file mode 100644 index b541c73db..000000000 --- a/pkg/services/util/response/client_stream.go +++ /dev/null @@ -1,48 +0,0 @@ -package response - -import ( - "context" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" -) - -// ClientMessageStreamer represents client-side message streamer -// that sets meta values to the response. -type ClientMessageStreamer struct { - cfg *cfg - - send util.RequestMessageWriter - - close util.ClientStreamCloser -} - -// Send calls send method of internal streamer. -func (s *ClientMessageStreamer) Send(ctx context.Context, req any) error { - if err := s.send(ctx, req); err != nil { - return fmt.Errorf("(%T) could not send the request: %w", s, err) - } - return nil -} - -// CloseAndRecv closes internal stream, receivers the response, -// sets meta values and returns the result. -func (s *ClientMessageStreamer) CloseAndRecv(ctx context.Context) (util.ResponseMessage, error) { - resp, err := s.close(ctx) - if err != nil { - return nil, fmt.Errorf("(%T) could not close stream and receive response: %w", s, err) - } - - setMeta(resp, s.cfg) - - return resp, nil -} - -// CreateRequestStreamer wraps stream methods and returns ClientMessageStreamer instance. -func (s *Service) CreateRequestStreamer(sender util.RequestMessageWriter, closer util.ClientStreamCloser) *ClientMessageStreamer { - return &ClientMessageStreamer{ - cfg: s.cfg, - send: sender, - close: closer, - } -} diff --git a/pkg/services/util/response/server_stream.go b/pkg/services/util/response/server_stream.go deleted file mode 100644 index 8a19fc4e7..000000000 --- a/pkg/services/util/response/server_stream.go +++ /dev/null @@ -1,37 +0,0 @@ -package response - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" -) - -// ServerMessageStreamer represents server-side message streamer -// that sets meta values to all response messages. -type ServerMessageStreamer struct { - cfg *cfg - - recv util.ResponseMessageReader -} - -// Recv calls Recv method of internal streamer, sets response meta -// values and returns the response. -func (s *ServerMessageStreamer) Recv() (util.ResponseMessage, error) { - m, err := s.recv() - if err != nil { - return nil, fmt.Errorf("could not receive response message for signing: %w", err) - } - - setMeta(m, s.cfg) - - return m, nil -} - -// HandleServerStreamRequest builds internal streamer via handlers, wraps it to ServerMessageStreamer and returns the result. -func (s *Service) HandleServerStreamRequest(respWriter util.ResponseMessageWriter) util.ResponseMessageWriter { - return func(resp util.ResponseMessage) error { - setMeta(resp, s.cfg) - - return respWriter(resp) - } -} diff --git a/pkg/services/util/response/service.go b/pkg/services/util/response/service.go index 87cc8383d..005a643e5 100644 --- a/pkg/services/util/response/service.go +++ b/pkg/services/util/response/service.go @@ -11,44 +11,24 @@ import ( // Service represents universal v2 service // that sets response meta header values. type Service struct { - cfg *cfg -} - -// Option is an option of Service constructor. -type Option func(*cfg) - -type cfg struct { version refs.Version state netmap.State } -func defaultCfg() *cfg { - var c cfg - - version.Current().WriteToV2(&c.version) - - return &c -} - // NewService creates, initializes and returns Service instance. -func NewService(opts ...Option) *Service { - c := defaultCfg() - - for i := range opts { - opts[i](c) - } - - return &Service{ - cfg: c, - } +func NewService(nmState netmap.State) *Service { + s := &Service{state: nmState} + version.Current().WriteToV2(&s.version) + return s } -func setMeta(resp util.ResponseMessage, cfg *cfg) { +// SetMeta sets adds meta-header to resp. +func (s *Service) SetMeta(resp util.ResponseMessage) { meta := new(session.ResponseMetaHeader) - meta.SetVersion(&cfg.version) + meta.SetVersion(&s.version) meta.SetTTL(1) // FIXME: #1160 TTL must be calculated - meta.SetEpoch(cfg.state.CurrentEpoch()) + meta.SetEpoch(s.state.CurrentEpoch()) if origin := resp.GetMetaHeader(); origin != nil { // FIXME: #1160 what if origin is set by local server? @@ -57,10 +37,3 @@ func setMeta(resp util.ResponseMessage, cfg *cfg) { resp.SetMetaHeader(meta) } - -// WithNetworkState returns option to set network state of Service. -func WithNetworkState(v netmap.State) Option { - return func(c *cfg) { - c.state = v - } -} diff --git a/pkg/services/util/response/unary.go b/pkg/services/util/response/unary.go deleted file mode 100644 index 29cb95314..000000000 --- a/pkg/services/util/response/unary.go +++ /dev/null @@ -1,21 +0,0 @@ -package response - -import ( - "context" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" -) - -// HandleUnaryRequest call passes request to handler, sets response meta header values and returns it. -func (s *Service) HandleUnaryRequest(ctx context.Context, req any, handler util.UnaryHandler) (util.ResponseMessage, error) { - // process request - resp, err := handler(ctx, req) - if err != nil { - return nil, fmt.Errorf("could not handle request: %w", err) - } - - setMeta(resp, s.cfg) - - return resp, nil -} diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index cb4be3084..a26bd311c 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -1,7 +1,6 @@ package util import ( - "context" "crypto/ecdsa" "errors" "fmt" @@ -21,198 +20,59 @@ type ResponseMessage interface { SetMetaHeader(*session.ResponseMetaHeader) } -type UnaryHandler func(context.Context, any) (ResponseMessage, error) - type SignService struct { key *ecdsa.PrivateKey } -type ResponseMessageWriter func(ResponseMessage) error - -type ServerStreamHandler func(context.Context, any) (ResponseMessageReader, error) - -type ResponseMessageReader func() (ResponseMessage, error) - var ErrAbortStream = errors.New("abort message stream") -type ResponseConstructor func() ResponseMessage - -type RequestMessageWriter func(context.Context, any) error - -type ClientStreamCloser func(context.Context) (ResponseMessage, error) - -type RequestMessageStreamer struct { - key *ecdsa.PrivateKey - - send RequestMessageWriter - - close ClientStreamCloser - - respCons ResponseConstructor - - statusSupported bool - - sendErr error -} - func NewUnarySignService(key *ecdsa.PrivateKey) *SignService { return &SignService{ key: key, } } -func (s *RequestMessageStreamer) Send(ctx context.Context, req any) error { - // req argument should be strengthen with type RequestMessage - s.statusSupported = isStatusSupported(req.(RequestMessage)) // panic is OK here for now - - var err error - - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { - err = fmt.Errorf("could not verify request: %w", err) - } else { - err = s.send(ctx, req) - } - - if err != nil { - if !s.statusSupported { - return err - } - - s.sendErr = err - - return ErrAbortStream - } - - return nil -} - -func (s *RequestMessageStreamer) CloseAndRecv(ctx context.Context) (ResponseMessage, error) { - var ( - resp ResponseMessage - err error - ) - - if s.sendErr != nil { - err = s.sendErr - } else { - resp, err = s.close(ctx) - if err != nil { - err = fmt.Errorf("could not close stream and receive response: %w", err) - } - } - - if err != nil { - if !s.statusSupported { - return nil, err - } - - resp = s.respCons() - - setStatusV2(resp, err) - } - - if err = signResponse(s.key, resp, s.statusSupported); err != nil { - return nil, err - } - - return resp, nil -} - -func (s *SignService) CreateRequestStreamer(sender RequestMessageWriter, closer ClientStreamCloser, blankResp ResponseConstructor) *RequestMessageStreamer { - return &RequestMessageStreamer{ - key: s.key, - send: sender, - close: closer, - - respCons: blankResp, - } -} - -func (s *SignService) HandleServerStreamRequest( - req any, - respWriter ResponseMessageWriter, - blankResp ResponseConstructor, - respWriterCaller func(ResponseMessageWriter) error, -) error { - // handle protocol versions <=2.10 (API statuses was introduced in 2.11 only) - - // req argument should be strengthen with type RequestMessage - statusSupported := isStatusSupported(req.(RequestMessage)) // panic is OK here for now - - var err error - - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { - err = fmt.Errorf("could not verify request: %w", err) - } else { - err = respWriterCaller(func(resp ResponseMessage) error { - if err := signResponse(s.key, resp, statusSupported); err != nil { - return err - } - - return respWriter(resp) - }) - } - +// SignResponse response with private key via signature.SignServiceMessage. +// The signature error affects the result depending on the protocol version: +// - if status return is supported, panics since we cannot return the failed status, because it will not be signed. +// - otherwise, returns error in order to transport it directly. +func (s *SignService) SignResponse(statusSupported bool, resp ResponseMessage, err error) error { if err != nil { if !statusSupported { return err } - resp := blankResp() - setStatusV2(resp, err) + } - _ = signResponse(s.key, resp, false) // panics or returns nil with false arg - - return respWriter(resp) + err = signature.SignServiceMessage(s.key, resp) + if err != nil { + return fmt.Errorf("could not sign response: %w", err) } return nil } -func (s *SignService) HandleUnaryRequest(ctx context.Context, req any, handler UnaryHandler, blankResp ResponseConstructor) (ResponseMessage, error) { - // handle protocol versions <=2.10 (API statuses was introduced in 2.11 only) - - // req argument should be strengthen with type RequestMessage - statusSupported := isStatusSupported(req.(RequestMessage)) // panic is OK here for now - - var ( - resp ResponseMessage - err error - ) - - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { +func (s *SignService) VerifyRequest(req RequestMessage) error { + if err := signature.VerifyServiceMessage(req); err != nil { var sigErr apistatus.SignatureVerification sigErr.SetMessage(err.Error()) - - err = sigErr - } else { - // process request - resp, err = handler(ctx, req) + return sigErr } - - if err != nil { - if !statusSupported { - return nil, err - } - - resp = blankResp() - - setStatusV2(resp, err) - } - - // sign the response - if err = signResponse(s.key, resp, statusSupported); err != nil { - return nil, err - } - - return resp, nil + return nil } -func isStatusSupported(req RequestMessage) bool { +// EnsureNonNilResponse creates an appropriate response struct if it is nil. +func EnsureNonNilResponse[T any](resp *T, err error) (*T, error) { + if resp != nil { + return resp, err + } + return new(T), err +} + +// IsStatusSupported returns true iff request version implies expecting status return. +// This allows us to handle protocol versions <=2.10 (API statuses was introduced in 2.11 only). +func IsStatusSupported(req RequestMessage) bool { version := req.GetMetaHeader().GetVersion() mjr := version.GetMajor() @@ -228,22 +88,3 @@ func setStatusV2(resp ResponseMessage, err error) { session.SetStatus(resp, apistatus.ToStatusV2(apistatus.ErrToStatus(err))) } - -// signs response with private key via signature.SignServiceMessage. -// The signature error affects the result depending on the protocol version: -// - if status return is supported, panics since we cannot return the failed status, because it will not be signed; -// - otherwise, returns error in order to transport it directly. -func signResponse(key *ecdsa.PrivateKey, resp any, statusSupported bool) error { - err := signature.SignServiceMessage(key, resp) - if err != nil { - err = fmt.Errorf("could not sign response: %w", err) - - if statusSupported { - // We can't pass this error as status code since response will be unsigned. - // Isn't expected in practice, so panic is ok here. - panic(err) - } - } - - return err -} diff --git a/pkg/util/attributes/parser.go b/pkg/util/attributes/parser.go index 8016c0160..f8cc97189 100644 --- a/pkg/util/attributes/parser.go +++ b/pkg/util/attributes/parser.go @@ -1,7 +1,6 @@ package attributes import ( - "errors" "fmt" "strings" @@ -21,7 +20,7 @@ func ReadNodeAttributes(dst *netmap.NodeInfo, attrs []string) error { k, v, found := strings.Cut(line, keyValueSeparator) if !found { - return errors.New("missing attribute key and/or value") + return fmt.Errorf("wrong format for node attribute: '%s'", attrs[i]) } _, ok := cache[k] @@ -36,9 +35,9 @@ func ReadNodeAttributes(dst *netmap.NodeInfo, attrs []string) error { v = replaceEscaping(v, true) if k == "" { - return errors.New("empty key") + return fmt.Errorf("empty key in node attribute: '%s'", attrs[i]) } else if v == "" { - return errors.New("empty value") + return fmt.Errorf("empty value in node attribute: '%s'", attrs[i]) } dst.SetAttribute(k, v) diff --git a/pkg/util/locode/db/boltdb/calls.go b/pkg/util/locode/db/boltdb/calls.go index 171808af2..6a80def3a 100644 --- a/pkg/util/locode/db/boltdb/calls.go +++ b/pkg/util/locode/db/boltdb/calls.go @@ -103,7 +103,7 @@ func recordFromValue(data []byte) (*locodedb.Record, error) { // Must not be called before successful Open call. // Must not be called in read-only mode: behavior is undefined. func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error { - return db.bolt.Update(func(tx *bbolt.Tx) error { + return db.bolt.Batch(func(tx *bbolt.Tx) error { countryKey, err := countryBucketKey(key.CountryCode()) if err != nil { return err diff --git a/pkg/util/locode/db/db.go b/pkg/util/locode/db/db.go index 2a0f26689..8c71ea794 100644 --- a/pkg/util/locode/db/db.go +++ b/pkg/util/locode/db/db.go @@ -3,8 +3,10 @@ package locodedb import ( "errors" "fmt" + "runtime" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" + "golang.org/x/sync/errgroup" ) // SourceTable is an interface of the UN/LOCODE table. @@ -75,81 +77,93 @@ type NamesDB interface { // FillDatabase generates the FrostFS location database based on the UN/LOCODE table. func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { - return table.IterateAll(func(tableRecord locode.Record) error { - if tableRecord.LOCODE.LocationCode() == "" { + var errG errgroup.Group + + // Pick some sane default, after this the performance stopped increasing. + errG.SetLimit(runtime.NumCPU() * 4) + _ = table.IterateAll(func(tableRecord locode.Record) error { + errG.Go(func() error { + return processTableRecord(tableRecord, airports, continents, names, db) + }) + return nil + }) + return errG.Wait() +} + +func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { + if tableRecord.LOCODE.LocationCode() == "" { + return nil + } + + dbKey, err := NewKey(tableRecord.LOCODE) + if err != nil { + return err + } + + dbRecord, err := NewRecord(tableRecord) + if err != nil { + if errors.Is(err, errParseCoordinates) { return nil } - dbKey, err := NewKey(tableRecord.LOCODE) - if err != nil { - return err - } + return err + } - dbRecord, err := NewRecord(tableRecord) + geoPoint := dbRecord.GeoPoint() + countryName := "" + + if geoPoint == nil { + airportRecord, err := airports.Get(tableRecord) if err != nil { - if errors.Is(err, errParseCoordinates) { + if errors.Is(err, ErrAirportNotFound) { return nil } return err } - geoPoint := dbRecord.GeoPoint() - countryName := "" + geoPoint = airportRecord.Point + countryName = airportRecord.CountryName + } - if geoPoint == nil { - airportRecord, err := airports.Get(tableRecord) - if err != nil { - if errors.Is(err, ErrAirportNotFound) { - return nil - } + dbRecord.SetGeoPoint(geoPoint) - return err - } - - geoPoint = airportRecord.Point - countryName = airportRecord.CountryName - } - - dbRecord.SetGeoPoint(geoPoint) - - if countryName == "" { - countryName, err = names.CountryName(dbKey.CountryCode()) - if err != nil { - if errors.Is(err, ErrCountryNotFound) { - return nil - } - - return err - } - } - - dbRecord.SetCountryName(countryName) - - if subDivCode := dbRecord.SubDivCode(); subDivCode != "" { - subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode) - if err != nil { - if errors.Is(err, ErrSubDivNotFound) { - return nil - } - - return err - } - - dbRecord.SetSubDivName(subDivName) - } - - continent, err := continents.PointContinent(geoPoint) + if countryName == "" { + countryName, err = names.CountryName(dbKey.CountryCode()) if err != nil { - return fmt.Errorf("could not calculate continent geo point: %w", err) - } else if continent.Is(ContinentUnknown) { - return nil + if errors.Is(err, ErrCountryNotFound) { + return nil + } + + return err + } + } + + dbRecord.SetCountryName(countryName) + + if subDivCode := dbRecord.SubDivCode(); subDivCode != "" { + subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode) + if err != nil { + if errors.Is(err, ErrSubDivNotFound) { + return nil + } + + return err } - dbRecord.SetContinent(continent) + dbRecord.SetSubDivName(subDivName) + } - return db.Put(*dbKey, *dbRecord) - }) + continent, err := continents.PointContinent(geoPoint) + if err != nil { + return fmt.Errorf("could not calculate continent geo point: %w", err) + } else if continent.Is(ContinentUnknown) { + return nil + } + + dbRecord.SetContinent(continent) + + return db.Put(*dbKey, *dbRecord) } // LocodeRecord returns the record from the FrostFS location database diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 4a536368a..fcac09321 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -31,6 +31,9 @@ type Prm struct { // support runtime rereading level zapcore.Level + // MetricsNamespace is the namespace string used for log counter metrics + MetricsNamespace string + // do not support runtime rereading } @@ -79,10 +82,15 @@ func NewLogger(prm *Prm) (*Logger, error) { lvl := zap.NewAtomicLevelAt(prm.level) + m := newLogMetrics(prm.MetricsNamespace) + c := zap.NewProductionConfig() c.Level = lvl c.Encoding = "console" c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + c.Sampling.Hook = func(e zapcore.Entry, sd zapcore.SamplingDecision) { + m.Inc(e.Level, sd == zapcore.LogDropped) + } lZap, err := c.Build( zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), diff --git a/pkg/util/logger/metrics.go b/pkg/util/logger/metrics.go new file mode 100644 index 000000000..708583473 --- /dev/null +++ b/pkg/util/logger/metrics.go @@ -0,0 +1,37 @@ +package logger + +import ( + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap/zapcore" +) + +const ( + logSubsystem = "logger" + logLevelLabel = "level" + logDroppedLabel = "dropped" +) + +type logMetrics struct { + logCount *prometheus.CounterVec +} + +func newLogMetrics(namespace string) *logMetrics { + return &logMetrics{ + logCount: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: logSubsystem, + Name: "entry_count", + Help: "Total log entries emitted or dropped by severity level", + }, []string{logLevelLabel, logDroppedLabel}), + } +} + +func (m *logMetrics) Inc(level zapcore.Level, dropped bool) { + m.logCount.With(prometheus.Labels{ + logLevelLabel: level.String(), + logDroppedLabel: strconv.FormatBool(dropped), + }).Inc() +} diff --git a/pkg/util/worker_pool.go b/pkg/util/worker_pool.go index 145fd1a5a..97d76c492 100644 --- a/pkg/util/worker_pool.go +++ b/pkg/util/worker_pool.go @@ -1,8 +1,9 @@ package util import ( + "sync/atomic" + "github.com/panjf2000/ants/v2" - "go.uber.org/atomic" ) // WorkerPool represents a tool to control diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index ac6e786ab..f29eca37c 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -4,10 +4,10 @@ import ( "encoding/json" "flag" "fmt" - "io/ioutil" "os" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) var ( @@ -26,10 +26,10 @@ func main() { var filename string switch { case *node != "": - _ = metrics.NewNodeMetrics() + _ = local_metrics.NewNodeMetrics() filename = *node case *ir != "": - _ = metrics.NewInnerRingMetrics() + _ = local_metrics.NewInnerRingMetrics() filename = *ir default: @@ -37,11 +37,7 @@ func main() { os.Exit(1) } - ds, err := metrics.DescribeAll() - if err != nil { - fmt.Fprintf(os.Stderr, "Could not parse metric descriptions: %v\n", err) - os.Exit(1) - } + ds := metrics.DescribeAll() data, err := json.Marshal(ds) if err != nil { @@ -49,7 +45,7 @@ func main() { os.Exit(1) } - if err := ioutil.WriteFile(filename, data, 0644); err != nil { + if err := os.WriteFile(filename, data, 0644); err != nil { fmt.Fprintf(os.Stderr, "Could write to file: %v\n", err) os.Exit(1) }