forked from TrueCloudLab/frostfs-node
Compare commits
37 commits
87ce7722c7
...
dbc91a4570
Author | SHA1 | Date | |
---|---|---|---|
dbc91a4570 | |||
35f65dea2e | |||
ad01fb958a | |||
d336f2d487 | |||
c82c753e9f | |||
f666898e5d | |||
b1a31281e4 | |||
764450d04a | |||
755cae3f19 | |||
9b13a18aac | |||
ef64930fef | |||
c8fb154151 | |||
7edec9193c | |||
3cf6ea745d | |||
9a77527f46 | |||
5b1ba8e23d | |||
9902965ff4 | |||
33ad753302 | |||
15102e6dfd | |||
17ec84151b | |||
6c45a17af6 | |||
d19ab43500 | |||
5bcf81d1cc | |||
c2effcc61c | |||
2285cfc36f | |||
e74d05c03f | |||
48862e0e63 | |||
89892d9754 | |||
7ac0852364 | |||
d28a5d2d7a | |||
87ac3c5279 | |||
d5ee6d3039 | |||
433aab12bb | |||
81f4cdbb91 | |||
3cd7d23f10 | |||
012af5cc38 | |||
eb5336d5ff |
247 changed files with 1510 additions and 1669 deletions
Before Width: | Height: | Size: 5.5 KiB After Width: | Height: | Size: 5.5 KiB |
|
@ -87,5 +87,7 @@ linters:
|
|||
- perfsprint
|
||||
- testifylint
|
||||
- protogetter
|
||||
- intrange
|
||||
- tenv
|
||||
disable-all: true
|
||||
fast: false
|
||||
|
|
|
@ -1,11 +0,0 @@
|
|||
pipeline:
|
||||
# Kludge for non-root containers under WoodPecker
|
||||
fix-ownership:
|
||||
image: alpine:latest
|
||||
commands: chown -R 1234:1234 .
|
||||
|
||||
pre-commit:
|
||||
image: git.frostfs.info/truecloudlab/frostfs-ci:v0.36
|
||||
commands:
|
||||
- export HOME="$(getent passwd $(id -u) | cut '-d:' -f6)"
|
||||
- pre-commit run --hook-stage manual
|
6
Makefile
6
Makefile
|
@ -8,10 +8,10 @@ HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs
|
|||
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||
|
||||
GO_VERSION ?= 1.22
|
||||
LINT_VERSION ?= 1.60.3
|
||||
LINT_VERSION ?= 1.61.0
|
||||
TRUECLOUDLAB_LINT_VERSION ?= 0.0.7
|
||||
PROTOC_VERSION ?= 25.0
|
||||
PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2)
|
||||
PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-sdk-go)
|
||||
PROTOC_OS_VERSION=osx-x86_64
|
||||
ifeq ($(shell uname), Linux)
|
||||
PROTOC_OS_VERSION=linux-x86_64
|
||||
|
@ -121,7 +121,7 @@ protoc-install:
|
|||
@unzip -q -o $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip -d $(PROTOC_DIR)
|
||||
@rm $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip
|
||||
@echo "⇒ Instaling protogen FrostFS plugin..."
|
||||
@GOBIN=$(PROTOGEN_FROSTFS_DIR) go install -mod=mod -v git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/protogen@$(PROTOGEN_FROSTFS_VERSION)
|
||||
@GOBIN=$(PROTOGEN_FROSTFS_DIR) go install -mod=mod -v git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/protogen@$(PROTOGEN_FROSTFS_VERSION)
|
||||
|
||||
# Build FrostFS component's docker image
|
||||
image-%:
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
<p align="center">
|
||||
<img src="./.github/logo.svg" width="500px" alt="FrostFS">
|
||||
<img src="./.forgejo/logo.svg" width="500px" alt="FrostFS">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
|
|
|
@ -128,7 +128,7 @@ func generateConfigExample(appDir string, credSize int) (string, error) {
|
|||
tmpl.AlphabetDir = filepath.Join(appDir, "alphabet-wallets")
|
||||
|
||||
var i innerring.GlagoliticLetter
|
||||
for i = 0; i < innerring.GlagoliticLetter(credSize); i++ {
|
||||
for i = range innerring.GlagoliticLetter(credSize) {
|
||||
tmpl.Glagolitics = append(tmpl.Glagolitics, i.String())
|
||||
}
|
||||
|
||||
|
|
|
@ -139,7 +139,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *he
|
|||
c, err := helper.GetN3Client(viper.GetViper())
|
||||
commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err)
|
||||
|
||||
ac, err := helper.NewLocalActor(cmd, c)
|
||||
ac, err := helper.NewLocalActor(cmd, c, constants.ConsensusAccountName)
|
||||
commonCmd.ExitOnErr(cmd, "can't create actor: %w", err)
|
||||
|
||||
var ch util.Uint160
|
||||
|
|
|
@ -63,7 +63,7 @@ func TestGenerateAlphabet(t *testing.T) {
|
|||
buf.Reset()
|
||||
v.Set(commonflags.AlphabetWalletsFlag, walletDir)
|
||||
require.NoError(t, GenerateAlphabetCmd.Flags().Set(commonflags.AlphabetSizeFlag, strconv.FormatUint(size, 10)))
|
||||
for i := uint64(0); i < size; i++ {
|
||||
for i := range uint64(size) {
|
||||
buf.WriteString(strconv.FormatUint(i, 10) + "\r")
|
||||
}
|
||||
|
||||
|
|
|
@ -5,7 +5,6 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"github.com/google/uuid"
|
||||
"github.com/nspcc-dev/neo-go/pkg/core/state"
|
||||
|
@ -31,7 +30,11 @@ type LocalActor struct {
|
|||
|
||||
// NewLocalActor create LocalActor with accounts form provided wallets.
|
||||
// In case of empty wallets provided created actor with dummy account only for read operation.
|
||||
func NewLocalActor(cmd *cobra.Command, c actor.RPCActor) (*LocalActor, error) {
|
||||
//
|
||||
// If wallets are provided, the contract client will use accounts with accName name from these wallets.
|
||||
// To determine which account name should be used in a contract client, refer to how the contract
|
||||
// verifies the transaction signature.
|
||||
func NewLocalActor(cmd *cobra.Command, c actor.RPCActor, accName string) (*LocalActor, error) {
|
||||
walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag))
|
||||
var act *actor.Actor
|
||||
var accounts []*wallet.Account
|
||||
|
@ -53,8 +56,8 @@ func NewLocalActor(cmd *cobra.Command, c actor.RPCActor) (*LocalActor, error) {
|
|||
commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err)
|
||||
|
||||
for _, w := range wallets {
|
||||
acc, err := GetWalletAccount(w, constants.CommitteeAccountName)
|
||||
commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err)
|
||||
acc, err := GetWalletAccount(w, accName)
|
||||
commonCmd.ExitOnErr(cmd, fmt.Sprintf("can't find %s account: %%w", accName), err)
|
||||
accounts = append(accounts, acc)
|
||||
}
|
||||
act, err = actor.New(c, []actor.SignerAccount{{
|
||||
|
|
|
@ -2,6 +2,7 @@ package nns
|
|||
|
||||
import (
|
||||
client "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/nns"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"github.com/nspcc-dev/neo-go/pkg/rpcclient/management"
|
||||
|
@ -15,7 +16,7 @@ func getRPCClient(cmd *cobra.Command) (*client.Contract, *helper.LocalActor, uti
|
|||
c, err := helper.GetN3Client(v)
|
||||
commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err)
|
||||
|
||||
ac, err := helper.NewLocalActor(cmd, c)
|
||||
ac, err := helper.NewLocalActor(cmd, c, constants.CommitteeAccountName)
|
||||
commonCmd.ExitOnErr(cmd, "can't create actor: %w", err)
|
||||
|
||||
r := management.NewReader(ac.Invoker)
|
||||
|
|
|
@ -659,9 +659,7 @@ func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes
|
|||
|
||||
for {
|
||||
n, ok = rdr.Read(buf)
|
||||
for i := range n {
|
||||
list = append(list, buf[i])
|
||||
}
|
||||
list = append(list, buf[:n]...)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
@ -672,9 +670,8 @@ func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes
|
|||
return nil, fmt.Errorf("read object list: %w", err)
|
||||
}
|
||||
|
||||
sort.Slice(list, func(i, j int) bool {
|
||||
lhs, rhs := list[i].EncodeToString(), list[j].EncodeToString()
|
||||
return strings.Compare(lhs, rhs) < 0
|
||||
slices.SortFunc(list, func(a, b oid.ID) int {
|
||||
return strings.Compare(a.EncodeToString(), b.EncodeToString())
|
||||
})
|
||||
|
||||
return &SearchObjectsRes{
|
||||
|
|
|
@ -2,7 +2,7 @@ package common
|
|||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
|
@ -45,15 +45,11 @@ func StartClientCommandSpan(cmd *cobra.Command) {
|
|||
})
|
||||
commonCmd.ExitOnErr(cmd, "init tracing: %w", err)
|
||||
|
||||
var components sort.StringSlice
|
||||
var components []string
|
||||
for c := cmd; c != nil; c = c.Parent() {
|
||||
components = append(components, c.Name())
|
||||
}
|
||||
for i, j := 0, len(components)-1; i < j; {
|
||||
components.Swap(i, j)
|
||||
i++
|
||||
j--
|
||||
}
|
||||
slices.Reverse(components)
|
||||
|
||||
operation := strings.Join(components, ".")
|
||||
ctx, span := tracing.StartSpanFromContext(cmd.Context(), operation)
|
||||
|
|
|
@ -7,12 +7,12 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
containerApi "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
||||
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
containerApi "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
|
|
|
@ -4,11 +4,11 @@ import (
|
|||
"encoding/hex"
|
||||
"errors"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
package control
|
||||
|
||||
import (
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
package control
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
package control
|
||||
|
||||
import (
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
package control
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
|
@ -7,11 +7,11 @@ import (
|
|||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
package control
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
|
@ -3,11 +3,11 @@ package control
|
|||
import (
|
||||
"encoding/hex"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
|
@ -3,11 +3,11 @@ package control
|
|||
import (
|
||||
"os"
|
||||
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
|
@ -3,12 +3,12 @@ package control
|
|||
import (
|
||||
"os"
|
||||
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
|
||||
ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
package control
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
|
||||
ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/spf13/cobra"
|
||||
|
|
|
@ -4,11 +4,11 @@ import (
|
|||
"encoding/hex"
|
||||
"errors"
|
||||
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
|
||||
ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
package control
|
||||
|
||||
import (
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
|
||||
ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
|
@ -5,11 +5,11 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
"github.com/nspcc-dev/neo-go/cli/input"
|
||||
|
|
|
@ -7,10 +7,10 @@ import (
|
|||
"strconv"
|
||||
"text/tabwriter"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/spf13/cobra"
|
||||
"google.golang.org/grpc/codes"
|
||||
|
|
|
@ -3,10 +3,10 @@ package control
|
|||
import (
|
||||
"fmt"
|
||||
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/mr-tron/base58"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
|
@ -4,10 +4,10 @@ import (
|
|||
"encoding/hex"
|
||||
"errors"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
|
@ -6,12 +6,12 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
|
@ -7,11 +7,11 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/mr-tron/base58"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
|
@ -6,10 +6,10 @@ import (
|
|||
"slices"
|
||||
"strings"
|
||||
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/mr-tron/base58"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
|
@ -4,12 +4,12 @@ import (
|
|||
"crypto/sha256"
|
||||
"errors"
|
||||
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
|
@ -4,11 +4,11 @@ import (
|
|||
"crypto/ecdsa"
|
||||
"errors"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
|
||||
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto"
|
||||
"github.com/spf13/cobra"
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
package control
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"github.com/mr-tron/base58"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
|
@ -6,12 +6,12 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
|
||||
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
|
|
@ -7,12 +7,12 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
|
|
@ -195,7 +195,7 @@ func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, member
|
|||
prmHead.SetRawFlag(true) // to get an error instead of whole object
|
||||
|
||||
eg, egCtx := errgroup.WithContext(cmd.Context())
|
||||
for idx := range len(members) {
|
||||
for idx := range members {
|
||||
partObjID := members[idx]
|
||||
|
||||
eg.Go(func() error {
|
||||
|
|
|
@ -10,11 +10,11 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
|
|
|
@ -21,7 +21,6 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-node/misc"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/gendoc"
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
@ -112,14 +111,16 @@ func initConfig() {
|
|||
// Use config file from the flag.
|
||||
viper.SetConfigFile(cfgFile)
|
||||
} else {
|
||||
// Find home directory.
|
||||
home, err := homedir.Dir()
|
||||
commonCmd.ExitOnErr(rootCmd, "", err)
|
||||
|
||||
// Search config in `$HOME/.config/frostfs-cli/` with name "config.yaml"
|
||||
viper.AddConfigPath(filepath.Join(home, ".config", "frostfs-cli"))
|
||||
viper.SetConfigName("config")
|
||||
viper.SetConfigType("yaml")
|
||||
// Find config directory.
|
||||
configDir, err := os.UserConfigDir()
|
||||
if err != nil {
|
||||
common.PrintVerbose(rootCmd, "Get config dir: %s", err)
|
||||
} else {
|
||||
// Search config in `$XDG_CONFIG_HOME/frostfs-cli/` with name "config.yaml"
|
||||
viper.AddConfigPath(filepath.Join(configDir, "frostfs-cli"))
|
||||
viper.SetConfigName("config")
|
||||
viper.SetConfigType("yaml")
|
||||
}
|
||||
}
|
||||
|
||||
viper.SetEnvPrefix(envPrefix)
|
||||
|
|
|
@ -11,7 +11,7 @@ func DecodeOIDs(data []byte) ([]oid.ID, error) {
|
|||
size := r.ReadVarUint()
|
||||
oids := make([]oid.ID, size)
|
||||
|
||||
for i := uint64(0); i < size; i++ {
|
||||
for i := range size {
|
||||
if err := oids[i].Decode(r.ReadVarBytes()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -4,11 +4,11 @@ import (
|
|||
"context"
|
||||
"net"
|
||||
|
||||
accountingGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting/grpc"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance"
|
||||
accountingTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/accounting/grpc"
|
||||
accountingService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/accounting"
|
||||
accounting "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/accounting/morph"
|
||||
accountingGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/accounting/grpc"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
|
|
|
@ -3,11 +3,11 @@ package main
|
|||
import (
|
||||
"net"
|
||||
|
||||
apemanager_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager/grpc"
|
||||
ape_contract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/contract_storage"
|
||||
morph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
|
||||
apemanager_transport "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/apemanager/grpc"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/apemanager"
|
||||
apemanager_grpc "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/apemanager/grpc"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
|
|
|
@ -15,7 +15,6 @@ import (
|
|||
"syscall"
|
||||
"time"
|
||||
|
||||
netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
|
||||
apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/audit"
|
||||
|
@ -58,6 +57,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
objectService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object"
|
||||
getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/tombstone"
|
||||
tsourse "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/tombstone/source"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator"
|
||||
|
@ -69,6 +69,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/logging/lokicore"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
||||
netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
|
@ -109,6 +110,7 @@ type applicationConfiguration struct {
|
|||
|
||||
ObjectCfg struct {
|
||||
tombstoneLifetime uint64
|
||||
priorityMetrics []placement.Metric
|
||||
}
|
||||
|
||||
EngineCfg struct {
|
||||
|
@ -232,6 +234,15 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error {
|
|||
// Object
|
||||
|
||||
a.ObjectCfg.tombstoneLifetime = objectconfig.TombstoneLifetime(c)
|
||||
var pm []placement.Metric
|
||||
for _, raw := range objectconfig.Get(c).Priority() {
|
||||
m, err := placement.ParseMetric(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pm = append(pm, m)
|
||||
}
|
||||
a.ObjectCfg.priorityMetrics = pm
|
||||
|
||||
// Storage Engine
|
||||
|
||||
|
@ -1179,17 +1190,15 @@ func initObjectPool(cfg *config.Config) (pool cfgObjectRoutines) {
|
|||
return pool
|
||||
}
|
||||
|
||||
func (c *cfg) LocalNodeInfo() (*netmapV2.NodeInfo, error) {
|
||||
var res netmapV2.NodeInfo
|
||||
|
||||
func (c *cfg) LocalNodeInfo() *netmap.NodeInfo {
|
||||
var res netmap.NodeInfo
|
||||
ni, ok := c.cfgNetmap.state.getNodeInfo()
|
||||
if ok {
|
||||
ni.WriteToV2(&res)
|
||||
res = ni
|
||||
} else {
|
||||
c.cfgNodeInfo.localInfo.WriteToV2(&res)
|
||||
res = c.cfgNodeInfo.localInfo
|
||||
}
|
||||
|
||||
return &res, nil
|
||||
return &res
|
||||
}
|
||||
|
||||
// setContractNodeInfo rewrites local node info from the FrostFS network map.
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
package config_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
|
@ -38,8 +37,7 @@ func TestConfigEnv(t *testing.T) {
|
|||
|
||||
envName := strings.ToUpper(
|
||||
strings.Join([]string{config.EnvPrefix, section, name}, configViper.EnvSeparator))
|
||||
err := os.Setenv(envName, value)
|
||||
require.NoError(t, err)
|
||||
t.Setenv(envName, value)
|
||||
|
||||
c := configtest.EmptyConfig()
|
||||
|
||||
|
|
|
@ -10,10 +10,17 @@ type PutConfig struct {
|
|||
cfg *config.Config
|
||||
}
|
||||
|
||||
// GetConfig is a wrapper over "get" config section which provides access
|
||||
// to object get pipeline configuration of object service.
|
||||
type GetConfig struct {
|
||||
cfg *config.Config
|
||||
}
|
||||
|
||||
const (
|
||||
subsection = "object"
|
||||
|
||||
putSubsection = "put"
|
||||
getSubsection = "get"
|
||||
|
||||
// PutPoolSizeDefault is a default value of routine pool size to
|
||||
// process object.Put requests in object service.
|
||||
|
@ -56,3 +63,16 @@ func (g PutConfig) PoolSizeLocal() int {
|
|||
func (g PutConfig) SkipSessionTokenIssuerVerification() bool {
|
||||
return config.BoolSafe(g.cfg, "skip_session_token_issuer_verification")
|
||||
}
|
||||
|
||||
// Get returns structure that provides access to "get" subsection of
|
||||
// "object" section.
|
||||
func Get(c *config.Config) GetConfig {
|
||||
return GetConfig{
|
||||
c.Sub(subsection).Sub(getSubsection),
|
||||
}
|
||||
}
|
||||
|
||||
// Priority returns the value of "priority" config parameter.
|
||||
func (g GetConfig) Priority() []string {
|
||||
return config.StringSliceSafe(g.cfg, "priority")
|
||||
}
|
||||
|
|
|
@ -11,8 +11,6 @@ import (
|
|||
)
|
||||
|
||||
func fromFile(path string) *config.Config {
|
||||
os.Clearenv() // ENVs have priority over config files, so we do this in tests
|
||||
|
||||
return config.New(path, "", "")
|
||||
}
|
||||
|
||||
|
@ -40,15 +38,6 @@ func ForEachFileType(pref string, f func(*config.Config)) {
|
|||
|
||||
// ForEnvFileType creates config from `<pref>.env` file.
|
||||
func ForEnvFileType(t testing.TB, pref string, f func(*config.Config)) {
|
||||
envs := os.Environ()
|
||||
t.Cleanup(func() {
|
||||
os.Clearenv()
|
||||
for _, env := range envs {
|
||||
keyValue := strings.Split(env, "=")
|
||||
os.Setenv(keyValue[0], keyValue[1])
|
||||
}
|
||||
})
|
||||
|
||||
f(fromEnvFile(t, pref+".env"))
|
||||
}
|
||||
|
||||
|
@ -73,7 +62,6 @@ func loadEnv(t testing.TB, path string) {
|
|||
|
||||
v = strings.Trim(v, `"`)
|
||||
|
||||
err = os.Setenv(k, v)
|
||||
require.NoError(t, err, "can't set environment variable")
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -10,6 +10,8 @@ import (
|
|||
|
||||
const (
|
||||
subsection = "tree"
|
||||
|
||||
SyncBatchSizeDefault = 1000
|
||||
)
|
||||
|
||||
// TreeConfig is a wrapper over "tree" config section
|
||||
|
@ -74,6 +76,17 @@ func (c TreeConfig) SyncInterval() time.Duration {
|
|||
return config.DurationSafe(c.cfg, "sync_interval")
|
||||
}
|
||||
|
||||
// SyncBatchSize returns the value of "sync_batch_size"
|
||||
// config parameter from the "tree" section.
|
||||
//
|
||||
// Returns `SyncBatchSizeDefault` if config value is not specified.
|
||||
func (c TreeConfig) SyncBatchSize() int {
|
||||
if v := config.IntSafe(c.cfg, "sync_batch_size"); v > 0 {
|
||||
return int(v)
|
||||
}
|
||||
return SyncBatchSizeDefault
|
||||
}
|
||||
|
||||
// AuthorizedKeys parses and returns an array of "authorized_keys" config
|
||||
// parameter from "tree" section.
|
||||
//
|
||||
|
|
|
@ -44,6 +44,7 @@ func TestTreeSection(t *testing.T) {
|
|||
require.Equal(t, 32, treeSec.ReplicationWorkerCount())
|
||||
require.Equal(t, 5*time.Second, treeSec.ReplicationTimeout())
|
||||
require.Equal(t, time.Hour, treeSec.SyncInterval())
|
||||
require.Equal(t, 2000, treeSec.SyncBatchSize())
|
||||
require.Equal(t, expectedKeys, treeSec.AuthorizedKeys())
|
||||
}
|
||||
|
||||
|
|
|
@ -5,9 +5,9 @@ import (
|
|||
"context"
|
||||
"net"
|
||||
|
||||
containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc"
|
||||
morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics"
|
||||
containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
|
||||
frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid"
|
||||
cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
|
||||
|
@ -17,6 +17,7 @@ import (
|
|||
containerTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/container/grpc"
|
||||
containerService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container"
|
||||
containerMorph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/morph"
|
||||
containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container/grpc"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
|
||||
|
@ -42,7 +43,7 @@ func initContainerService(_ context.Context, c *cfg) {
|
|||
|
||||
cacheSize := morphconfig.FrostfsIDCacheSize(c.appCfg)
|
||||
if cacheSize > 0 {
|
||||
frostfsIDSubjectProvider = newMorphFrostfsIDCache(frostfsIDSubjectProvider, int(cacheSize), c.cfgMorph.cacheTTL)
|
||||
frostfsIDSubjectProvider = newMorphFrostfsIDCache(frostfsIDSubjectProvider, int(cacheSize), c.cfgMorph.cacheTTL, metrics.NewCacheMetrics("frostfs_id"))
|
||||
}
|
||||
|
||||
c.shared.frostfsidClient = frostfsIDSubjectProvider
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client"
|
||||
|
@ -9,57 +10,101 @@ import (
|
|||
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||
)
|
||||
|
||||
type subjectWithError struct {
|
||||
subject *client.Subject
|
||||
err error
|
||||
}
|
||||
|
||||
type subjectExtWithError struct {
|
||||
subject *client.SubjectExtended
|
||||
err error
|
||||
}
|
||||
|
||||
type morphFrostfsIDCache struct {
|
||||
subjProvider frostfsidcore.SubjectProvider
|
||||
|
||||
subjCache *expirable.LRU[util.Uint160, *client.Subject]
|
||||
subjCache *expirable.LRU[util.Uint160, subjectWithError]
|
||||
|
||||
subjExtCache *expirable.LRU[util.Uint160, *client.SubjectExtended]
|
||||
subjExtCache *expirable.LRU[util.Uint160, subjectExtWithError]
|
||||
|
||||
metrics cacheMetrics
|
||||
}
|
||||
|
||||
func newMorphFrostfsIDCache(subjProvider frostfsidcore.SubjectProvider, size int, ttl time.Duration) frostfsidcore.SubjectProvider {
|
||||
func newMorphFrostfsIDCache(subjProvider frostfsidcore.SubjectProvider, size int, ttl time.Duration, metrics cacheMetrics) frostfsidcore.SubjectProvider {
|
||||
return &morphFrostfsIDCache{
|
||||
subjProvider: subjProvider,
|
||||
|
||||
subjCache: expirable.NewLRU(size, func(util.Uint160, *client.Subject) {}, ttl),
|
||||
subjCache: expirable.NewLRU(size, func(util.Uint160, subjectWithError) {}, ttl),
|
||||
|
||||
subjExtCache: expirable.NewLRU(size, func(util.Uint160, *client.SubjectExtended) {}, ttl),
|
||||
subjExtCache: expirable.NewLRU(size, func(util.Uint160, subjectExtWithError) {}, ttl),
|
||||
|
||||
metrics: metrics,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *morphFrostfsIDCache) GetSubject(addr util.Uint160) (*client.Subject, error) {
|
||||
hit := false
|
||||
startedAt := time.Now()
|
||||
defer func() {
|
||||
m.metrics.AddMethodDuration("GetSubject", time.Since(startedAt), hit)
|
||||
}()
|
||||
|
||||
result, found := m.subjCache.Get(addr)
|
||||
if found {
|
||||
return result, nil
|
||||
hit = true
|
||||
return result.subject, result.err
|
||||
}
|
||||
|
||||
result, err := m.subjProvider.GetSubject(addr)
|
||||
subj, err := m.subjProvider.GetSubject(addr)
|
||||
if err != nil {
|
||||
if m.isCacheableError(err) {
|
||||
m.subjCache.Add(addr, subjectWithError{
|
||||
err: err,
|
||||
})
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m.subjCache.Add(addr, result)
|
||||
return result, nil
|
||||
m.subjCache.Add(addr, subjectWithError{subject: subj})
|
||||
return subj, nil
|
||||
}
|
||||
|
||||
func (m *morphFrostfsIDCache) GetSubjectExtended(addr util.Uint160) (*client.SubjectExtended, error) {
|
||||
subjExt, found := m.subjExtCache.Get(addr)
|
||||
hit := false
|
||||
startedAt := time.Now()
|
||||
defer func() {
|
||||
m.metrics.AddMethodDuration("GetSubjectExtended", time.Since(startedAt), hit)
|
||||
}()
|
||||
|
||||
result, found := m.subjExtCache.Get(addr)
|
||||
if found {
|
||||
return subjExt, nil
|
||||
hit = true
|
||||
return result.subject, result.err
|
||||
}
|
||||
|
||||
var err error
|
||||
subjExt, err = m.subjProvider.GetSubjectExtended(addr)
|
||||
subjExt, err := m.subjProvider.GetSubjectExtended(addr)
|
||||
if err != nil {
|
||||
if m.isCacheableError(err) {
|
||||
m.subjExtCache.Add(addr, subjectExtWithError{
|
||||
err: err,
|
||||
})
|
||||
m.subjCache.Add(addr, subjectWithError{
|
||||
err: err,
|
||||
})
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m.subjExtCache.Add(addr, subjExt)
|
||||
m.subjCache.Add(addr, subjectFromSubjectExtended(subjExt))
|
||||
m.subjExtCache.Add(addr, subjectExtWithError{subject: subjExt})
|
||||
m.subjCache.Add(addr, subjectWithError{subject: subjectFromSubjectExtended(subjExt)})
|
||||
|
||||
return subjExt, nil
|
||||
}
|
||||
|
||||
func (m *morphFrostfsIDCache) isCacheableError(err error) bool {
|
||||
return strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage)
|
||||
}
|
||||
|
||||
func subjectFromSubjectExtended(subjExt *client.SubjectExtended) *client.Subject {
|
||||
return &client.Subject{
|
||||
PrimaryKey: subjExt.PrimaryKey,
|
||||
|
|
|
@ -17,15 +17,16 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand"
|
||||
"github.com/nspcc-dev/neo-go/pkg/core/block"
|
||||
"github.com/nspcc-dev/neo-go/pkg/core/state"
|
||||
"github.com/nspcc-dev/neo-go/pkg/neorpc/result"
|
||||
"github.com/nspcc-dev/neo-go/pkg/rpcclient/waiter"
|
||||
"github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger"
|
||||
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||
"github.com/nspcc-dev/neo-go/pkg/vm/vmstate"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
newEpochNotification = "NewEpoch"
|
||||
|
||||
// amount of tries(blocks) before notary deposit timeout.
|
||||
notaryDepositRetriesAmount = 300
|
||||
)
|
||||
|
||||
func (c *cfg) initMorphComponents(ctx context.Context) {
|
||||
|
@ -128,7 +129,7 @@ func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) {
|
|||
return
|
||||
}
|
||||
|
||||
tx, err := makeNotaryDeposit(c)
|
||||
tx, vub, err := makeNotaryDeposit(c)
|
||||
fatalOnErr(err)
|
||||
|
||||
if tx.Equals(util.Uint256{}) {
|
||||
|
@ -139,11 +140,11 @@ func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) {
|
|||
return
|
||||
}
|
||||
|
||||
err = waitNotaryDeposit(ctx, c, tx)
|
||||
err = waitNotaryDeposit(ctx, c, tx, vub)
|
||||
fatalOnErr(err)
|
||||
}
|
||||
|
||||
func makeNotaryDeposit(c *cfg) (util.Uint256, error) {
|
||||
func makeNotaryDeposit(c *cfg) (util.Uint256, uint32, error) {
|
||||
const (
|
||||
// gasMultiplier defines how many times more the notary
|
||||
// balance must be compared to the GAS balance of the node:
|
||||
|
@ -157,7 +158,7 @@ func makeNotaryDeposit(c *cfg) (util.Uint256, error) {
|
|||
|
||||
depositAmount, err := client.CalculateNotaryDepositAmount(c.cfgMorph.client, gasMultiplier, gasDivisor)
|
||||
if err != nil {
|
||||
return util.Uint256{}, fmt.Errorf("could not calculate notary deposit: %w", err)
|
||||
return util.Uint256{}, 0, fmt.Errorf("could not calculate notary deposit: %w", err)
|
||||
}
|
||||
|
||||
return c.cfgMorph.client.DepositEndlessNotary(depositAmount)
|
||||
|
@ -168,32 +169,43 @@ var (
|
|||
errNotaryDepositTimeout = errors.New("notary deposit tx has not appeared in the network")
|
||||
)
|
||||
|
||||
func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
|
||||
for range notaryDepositRetriesAmount {
|
||||
c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
type waiterClient struct {
|
||||
c *client.Client
|
||||
}
|
||||
|
||||
ok, err := c.cfgMorph.client.TxHalt(tx)
|
||||
if err == nil {
|
||||
if ok {
|
||||
c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted)
|
||||
return nil
|
||||
}
|
||||
func (w *waiterClient) Context() context.Context {
|
||||
return context.Background()
|
||||
}
|
||||
|
||||
return errNotaryDepositFail
|
||||
}
|
||||
func (w *waiterClient) GetApplicationLog(hash util.Uint256, trig *trigger.Type) (*result.ApplicationLog, error) {
|
||||
return w.c.GetApplicationLog(hash, trig)
|
||||
}
|
||||
|
||||
err = c.cfgMorph.client.Wait(ctx, 1)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not wait for one block in chain: %w", err)
|
||||
}
|
||||
func (w *waiterClient) GetBlockCount() (uint32, error) {
|
||||
return w.c.BlockCount()
|
||||
}
|
||||
|
||||
func (w *waiterClient) GetVersion() (*result.Version, error) {
|
||||
return w.c.GetVersion()
|
||||
}
|
||||
|
||||
func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256, vub uint32) error {
|
||||
w, err := waiter.NewPollingBased(&waiterClient{c: c.cfgMorph.client})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create notary deposit waiter: %w", err)
|
||||
}
|
||||
|
||||
return errNotaryDepositTimeout
|
||||
res, err := w.WaitAny(ctx, vub, tx)
|
||||
if err != nil {
|
||||
if errors.Is(err, waiter.ErrTxNotAccepted) {
|
||||
return errNotaryDepositTimeout
|
||||
}
|
||||
return fmt.Errorf("could not wait for notary deposit persists in chain: %w", err)
|
||||
}
|
||||
if res.Execution.VMState.HasFlag(vmstate.Halt) {
|
||||
c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted)
|
||||
return nil
|
||||
}
|
||||
return errNotaryDepositFail
|
||||
}
|
||||
|
||||
func listenMorphNotifications(ctx context.Context, c *cfg) {
|
||||
|
|
|
@ -8,7 +8,6 @@ import (
|
|||
"net"
|
||||
"sync/atomic"
|
||||
|
||||
netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
|
||||
|
@ -19,6 +18,7 @@ import (
|
|||
netmapTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/netmap/grpc"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
netmapService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/netmap"
|
||||
netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap/grpc"
|
||||
netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
|
||||
"go.uber.org/zap"
|
||||
|
@ -192,7 +192,7 @@ func addNewEpochNotificationHandlers(c *cfg) {
|
|||
|
||||
if c.cfgMorph.notaryEnabled {
|
||||
addNewEpochAsyncNotificationHandler(c, func(_ event.Event) {
|
||||
_, err := makeNotaryDeposit(c)
|
||||
_, _, err := makeNotaryDeposit(c)
|
||||
if err != nil {
|
||||
c.log.Error(logs.FrostFSNodeCouldNotMakeNotaryDeposit,
|
||||
zap.String("error", err.Error()),
|
||||
|
|
|
@ -6,8 +6,6 @@ import (
|
|||
"fmt"
|
||||
"net"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object/grpc"
|
||||
metricsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/metrics"
|
||||
policerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/policer"
|
||||
replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator"
|
||||
|
@ -21,7 +19,6 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/cache"
|
||||
objectTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/object/grpc"
|
||||
objectService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl"
|
||||
v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2"
|
||||
objectAPE "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/ape"
|
||||
objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer"
|
||||
|
@ -38,8 +35,9 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
@ -178,7 +176,8 @@ func initObjectService(c *cfg) {
|
|||
|
||||
sSearchV2 := createSearchSvcV2(sSearch, keyStorage)
|
||||
|
||||
sGet := createGetService(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource)
|
||||
sGet := createGetService(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource,
|
||||
c.ObjectCfg.priorityMetrics)
|
||||
|
||||
*c.cfgObject.getSvc = *sGet // need smth better
|
||||
|
||||
|
@ -389,6 +388,7 @@ func createSearchSvcV2(sSearch *searchsvc.Service, keyStorage *util.KeyStorage)
|
|||
func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator,
|
||||
coreConstructor *cache.ClientCache,
|
||||
containerSource containercore.Source,
|
||||
priorityMetrics []placement.Metric,
|
||||
) *getsvc.Service {
|
||||
ls := c.cfgObject.cfgLocalStorage.localStorage
|
||||
|
||||
|
@ -398,6 +398,8 @@ func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Tra
|
|||
ls,
|
||||
traverseGen.WithTraverseOptions(
|
||||
placement.SuccessAfter(1),
|
||||
placement.WithPriorityMetrics(priorityMetrics),
|
||||
placement.WithNodeState(c),
|
||||
),
|
||||
coreConstructor,
|
||||
containerSource,
|
||||
|
@ -454,17 +456,10 @@ func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Servi
|
|||
}
|
||||
|
||||
func createACLServiceV2(c *cfg, apeSvc *objectAPE.Service, irFetcher *cachedIRFetcher) v2.Service {
|
||||
ls := c.cfgObject.cfgLocalStorage.localStorage
|
||||
|
||||
return v2.New(
|
||||
apeSvc,
|
||||
c.netMapSource,
|
||||
irFetcher,
|
||||
acl.NewChecker(
|
||||
c.cfgNetmap.state,
|
||||
c.cfgObject.eaclSource,
|
||||
eaclSDK.NewValidator(),
|
||||
ls),
|
||||
c.cfgObject.cnrSource,
|
||||
v2.WithLogger(c.log),
|
||||
)
|
||||
|
|
|
@ -6,8 +6,6 @@ import (
|
|||
"net"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session"
|
||||
sessionGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session/grpc"
|
||||
nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap"
|
||||
|
@ -16,6 +14,8 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/persistent"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/temporary"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session"
|
||||
sessionGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session/grpc"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
|
|
@ -62,6 +62,7 @@ func initTreeService(c *cfg) {
|
|||
tree.WithReplicationTimeout(treeConfig.ReplicationTimeout()),
|
||||
tree.WithReplicationChannelCapacity(treeConfig.ReplicationChannelCapacity()),
|
||||
tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount()),
|
||||
tree.WithSyncBatchSize(treeConfig.SyncBatchSize()),
|
||||
tree.WithAuthorizedKeys(treeConfig.AuthorizedKeys()),
|
||||
tree.WithMetrics(c.metricsCollector.TreeService()),
|
||||
tree.WithAPELocalOverrideStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalStorage()),
|
||||
|
|
|
@ -31,6 +31,7 @@ FROSTFS_TREE_REPLICATION_CHANNEL_CAPACITY=32
|
|||
FROSTFS_TREE_REPLICATION_WORKER_COUNT=32
|
||||
FROSTFS_TREE_REPLICATION_TIMEOUT=5s
|
||||
FROSTFS_TREE_SYNC_INTERVAL=1h
|
||||
FROSTFS_TREE_SYNC_BATCH_SIZE=2000
|
||||
FROSTFS_TREE_AUTHORIZED_KEYS="0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0 02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56"
|
||||
|
||||
# gRPC section
|
||||
|
@ -87,6 +88,7 @@ FROSTFS_OBJECT_PUT_REMOTE_POOL_SIZE=100
|
|||
FROSTFS_OBJECT_PUT_LOCAL_POOL_SIZE=200
|
||||
FROSTFS_OBJECT_PUT_SKIP_SESSION_TOKEN_ISSUER_VERIFICATION=true
|
||||
FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10
|
||||
FROSTFS_OBJECT_GET_PRIORITY="$attribute:ClusterName $attribute:UN-LOCODE"
|
||||
|
||||
# Storage engine section
|
||||
FROSTFS_STORAGE_SHARD_POOL_SIZE=15
|
||||
|
|
|
@ -69,6 +69,7 @@
|
|||
"replication_worker_count": 32,
|
||||
"replication_timeout": "5s",
|
||||
"sync_interval": "1h",
|
||||
"sync_batch_size": 2000,
|
||||
"authorized_keys": [
|
||||
"0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0",
|
||||
"02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56"
|
||||
|
@ -131,6 +132,9 @@
|
|||
"remote_pool_size": 100,
|
||||
"local_pool_size": 200,
|
||||
"skip_session_token_issuer_verification": true
|
||||
},
|
||||
"get": {
|
||||
"priority": ["$attribute:ClusterName", "$attribute:UN-LOCODE"]
|
||||
}
|
||||
},
|
||||
"storage": {
|
||||
|
|
|
@ -59,6 +59,7 @@ tree:
|
|||
replication_channel_capacity: 32
|
||||
replication_timeout: 5s
|
||||
sync_interval: 1h
|
||||
sync_batch_size: 2000
|
||||
authorized_keys: # list of hex-encoded public keys that have rights to use the Tree Service with frostfs-cli
|
||||
- 0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0
|
||||
- 02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56
|
||||
|
@ -114,6 +115,10 @@ object:
|
|||
remote_pool_size: 100 # number of async workers for remote PUT operations
|
||||
local_pool_size: 200 # number of async workers for local PUT operations
|
||||
skip_session_token_issuer_verification: true # session token issuer verification will be skipped if true
|
||||
get:
|
||||
priority: # list of metrics of nodes for prioritization
|
||||
- $attribute:ClusterName
|
||||
- $attribute:UN-LOCODE
|
||||
|
||||
storage:
|
||||
# note: shard configuration can be omitted for relay node (see `node.relay`)
|
||||
|
|
|
@ -55,7 +55,7 @@ Add an entry to the `CHANGELOG.md` following the style established there.
|
|||
* update `Unreleased...new` and `new...old` diff-links at the bottom of the file
|
||||
* add optional codename and release date in the heading
|
||||
* remove all empty sections such as `Added`, `Removed`, etc.
|
||||
* make sure all changes have references to GitHub issues in `#123` format (if possible)
|
||||
* make sure all changes have references to relevant issues in `#123` format (if possible)
|
||||
* clean up all `Unreleased` sections and leave them empty
|
||||
|
||||
### Make release commit
|
||||
|
@ -110,9 +110,9 @@ $ docker push truecloudlab/frostfs-cli:${FROSTFS_REVISION}
|
|||
$ docker push truecloudlab/frostfs-adm:${FROSTFS_REVISION}
|
||||
```
|
||||
|
||||
### Make a proper GitHub release (if not automated)
|
||||
### Make a proper release (if not automated)
|
||||
|
||||
Edit an automatically-created release on GitHub, copy things from `CHANGELOG.md`.
|
||||
Edit an automatically-created release on git.frostfs.info, copy things from `CHANGELOG.md`.
|
||||
Build and tar release binaries with `make prepare-release`, attach them to
|
||||
the release. Publish the release.
|
||||
|
||||
|
@ -121,7 +121,7 @@ the release. Publish the release.
|
|||
Prepare pull-request in [frostfs-devenv](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env)
|
||||
with new versions.
|
||||
|
||||
### Close GitHub milestone
|
||||
### Close milestone
|
||||
|
||||
Look up [milestones](https://git.frostfs.info/TrueCloudLab/frostfs-node/milestones) and close the release one if exists.
|
||||
|
||||
|
|
|
@ -407,13 +407,17 @@ Contains object-service related parameters.
|
|||
object:
|
||||
put:
|
||||
remote_pool_size: 100
|
||||
get:
|
||||
priority:
|
||||
- $attribute:ClusterName
|
||||
```
|
||||
|
||||
| Parameter | Type | Default value | Description |
|
||||
|-----------------------------|-------|---------------|------------------------------------------------------------------------------------------------|
|
||||
| `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. |
|
||||
| `put.remote_pool_size` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. |
|
||||
| `put.local_pool_size` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. |
|
||||
| Parameter | Type | Default value | Description |
|
||||
|-----------------------------|------------|---------------|------------------------------------------------------------------------------------------------------|
|
||||
| `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. |
|
||||
| `put.remote_pool_size` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. |
|
||||
| `put.local_pool_size` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. |
|
||||
| `get.priority` | `[]string` | | List of metrics of nodes for prioritization. Used for computing response on GET and SEARCH requests. |
|
||||
|
||||
# `runtime` section
|
||||
Contains runtime parameters.
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
## Update CI
|
||||
|
||||
Change Golang versions for unit test in CI.
|
||||
There is `go` section in `.github/workflows/go.yaml` file:
|
||||
There is `go` section in `.forgejo/workflows/*.yml` files:
|
||||
```yaml
|
||||
jobs:
|
||||
test:
|
||||
|
|
6
go.mod
6
go.mod
|
@ -4,12 +4,11 @@ go 1.22
|
|||
|
||||
require (
|
||||
code.gitea.io/sdk/gitea v0.17.1
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241007120543-29c522d5d8a3
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0
|
||||
git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241010110344-99c5c5836509
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241107121119-cb813e27a823
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1
|
||||
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972
|
||||
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88
|
||||
|
@ -19,6 +18,7 @@ require (
|
|||
github.com/cheggaaa/pb v1.0.29
|
||||
github.com/chzyer/readline v1.5.1
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
|
||||
github.com/felixge/fgprof v0.9.5
|
||||
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
|
||||
github.com/gdamore/tcell/v2 v2.7.4
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.0.0
|
||||
|
@ -26,7 +26,6 @@ require (
|
|||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/klauspost/compress v1.17.4
|
||||
github.com/mailru/easyjson v0.7.7
|
||||
github.com/mitchellh/go-homedir v1.1.0
|
||||
github.com/mr-tron/base58 v1.2.0
|
||||
github.com/multiformats/go-multiaddr v0.12.1
|
||||
github.com/nspcc-dev/neo-go v0.106.3
|
||||
|
@ -77,6 +76,7 @@ require (
|
|||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
|
||||
github.com/gorilla/websocket v1.5.1 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect
|
||||
|
|
BIN
go.sum
BIN
go.sum
Binary file not shown.
|
@ -4,7 +4,7 @@ import (
|
|||
"encoding/hex"
|
||||
"fmt"
|
||||
|
||||
v2acl "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
|
||||
v2acl "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/acl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native"
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
package audit
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session"
|
||||
crypto "git.frostfs.info/TrueCloudLab/frostfs-crypto"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
|
|
@ -3,7 +3,7 @@ package audit
|
|||
import (
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
|
|
@ -17,8 +17,6 @@ const (
|
|||
)
|
||||
|
||||
const (
|
||||
InnerringNonalphabetModeDoNotStopContainerEstimations = "non-alphabet mode, do not stop container estimations"
|
||||
InnerringCantStopEpochEstimation = "can't stop epoch estimation"
|
||||
InnerringCantMakeNotaryDepositInMainChain = "can't make notary deposit in main chain"
|
||||
InnerringCantMakeNotaryDepositInSideChain = "can't make notary deposit in side chain"
|
||||
InnerringNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made"
|
||||
|
@ -142,7 +140,6 @@ const (
|
|||
ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked"
|
||||
ClientNotaryRequestInvoked = "notary request invoked"
|
||||
ClientNotaryDepositTransactionWasSuccessfullyPersisted = "notary deposit transaction was successfully persisted"
|
||||
ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted = "attempt to wait for notary deposit transaction to get persisted"
|
||||
ClientNeoClientInvoke = "neo client invoke"
|
||||
ClientNativeGasTransferInvoke = "native gas transfer invoke"
|
||||
ClientBatchGasTransferInvoke = "batch gas transfer invoke"
|
||||
|
@ -344,7 +341,6 @@ const (
|
|||
NetmapCantGetTransactionHeight = "can't get transaction height"
|
||||
NetmapCantResetEpochTimer = "can't reset epoch timer"
|
||||
NetmapCantGetNetmapSnapshotToPerformCleanup = "can't get netmap snapshot to perform cleanup"
|
||||
NetmapCantStartContainerSizeEstimation = "can't start container size estimation"
|
||||
NetmapNonAlphabetModeIgnoreNewEpochTick = "non alphabet mode, ignore new epoch tick"
|
||||
NetmapNextEpoch = "next epoch"
|
||||
NetmapCantInvokeNetmapNewEpoch = "can't invoke netmap.NewEpoch"
|
||||
|
|
|
@ -3,8 +3,8 @@ package client
|
|||
import (
|
||||
"context"
|
||||
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
|
||||
rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
)
|
||||
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
package container
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
)
|
||||
|
|
|
@ -8,11 +8,11 @@ import (
|
|||
"fmt"
|
||||
"strconv"
|
||||
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
|
|
|
@ -7,9 +7,9 @@ import (
|
|||
"strconv"
|
||||
"testing"
|
||||
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
|
|
|
@ -3,14 +3,10 @@ package innerring
|
|||
import (
|
||||
"context"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet"
|
||||
timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/timer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
|
||||
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type (
|
||||
|
@ -19,28 +15,12 @@ type (
|
|||
EpochDuration() uint64
|
||||
}
|
||||
|
||||
alphaState interface {
|
||||
IsAlphabet() bool
|
||||
}
|
||||
|
||||
newEpochHandler func()
|
||||
|
||||
containerEstimationStopper interface {
|
||||
StopEstimation(p container.StopEstimationPrm) error
|
||||
}
|
||||
|
||||
epochTimerArgs struct {
|
||||
l *logger.Logger
|
||||
|
||||
alphabetState alphaState
|
||||
|
||||
newEpochHandlers []newEpochHandler
|
||||
|
||||
cnrWrapper containerEstimationStopper // to invoke stop container estimation
|
||||
epoch epochState // to specify which epoch to stop, and epoch duration
|
||||
|
||||
stopEstimationDMul uint32 // X: X/Y of epoch in blocks
|
||||
stopEstimationDDiv uint32 // Y: X/Y of epoch in blocks
|
||||
epoch epochState // to specify which epoch to stop, and epoch duration
|
||||
}
|
||||
|
||||
emitTimerArgs struct {
|
||||
|
@ -74,7 +54,7 @@ func (s *Server) tickTimers(h uint32) {
|
|||
}
|
||||
|
||||
func newEpochTimer(args *epochTimerArgs) *timer.BlockTimer {
|
||||
epochTimer := timer.NewBlockTimer(
|
||||
return timer.NewBlockTimer(
|
||||
func() (uint32, error) {
|
||||
return uint32(args.epoch.EpochDuration()), nil
|
||||
},
|
||||
|
@ -84,35 +64,6 @@ func newEpochTimer(args *epochTimerArgs) *timer.BlockTimer {
|
|||
}
|
||||
},
|
||||
)
|
||||
|
||||
// sub-timer for epoch timer to tick stop container estimation events at
|
||||
// some block in epoch
|
||||
epochTimer.OnDelta(
|
||||
args.stopEstimationDMul,
|
||||
args.stopEstimationDDiv,
|
||||
func() {
|
||||
if !args.alphabetState.IsAlphabet() {
|
||||
args.l.Debug(logs.InnerringNonalphabetModeDoNotStopContainerEstimations)
|
||||
return
|
||||
}
|
||||
|
||||
epochN := args.epoch.EpochCounter()
|
||||
if epochN == 0 { // estimates are invalid in genesis epoch
|
||||
return
|
||||
}
|
||||
|
||||
prm := container.StopEstimationPrm{}
|
||||
prm.SetEpoch(epochN - 1)
|
||||
|
||||
err := args.cnrWrapper.StopEstimation(prm)
|
||||
if err != nil {
|
||||
args.l.Warn(logs.InnerringCantStopEpochEstimation,
|
||||
zap.Uint64("epoch", epochN),
|
||||
zap.String("error", err.Error()))
|
||||
}
|
||||
})
|
||||
|
||||
return epochTimer
|
||||
}
|
||||
|
||||
func newEmissionTimer(args *emitTimerArgs) *timer.BlockTimer {
|
||||
|
|
|
@ -3,29 +3,20 @@ package innerring
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEpochTimer(t *testing.T) {
|
||||
t.Parallel()
|
||||
alphaState := &testAlphabetState{isAlphabet: true}
|
||||
neh := &testNewEpochHandler{}
|
||||
cnrStopper := &testContainerEstStopper{}
|
||||
epochState := &testEpochState{
|
||||
counter: 99,
|
||||
duration: 10,
|
||||
}
|
||||
|
||||
args := &epochTimerArgs{
|
||||
l: test.NewLogger(t),
|
||||
alphabetState: alphaState,
|
||||
newEpochHandlers: []newEpochHandler{neh.Handle},
|
||||
cnrWrapper: cnrStopper,
|
||||
epoch: epochState,
|
||||
stopEstimationDMul: 2,
|
||||
stopEstimationDDiv: 10,
|
||||
newEpochHandlers: []newEpochHandler{neh.Handle},
|
||||
epoch: epochState,
|
||||
}
|
||||
et := newEpochTimer(args)
|
||||
err := et.Reset()
|
||||
|
@ -33,63 +24,43 @@ func TestEpochTimer(t *testing.T) {
|
|||
|
||||
et.Tick(100)
|
||||
require.Equal(t, 0, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 0, cnrStopper.called, "invalid container stop handler calls")
|
||||
|
||||
et.Tick(101)
|
||||
require.Equal(t, 0, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls")
|
||||
|
||||
et.Tick(102)
|
||||
require.Equal(t, 0, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls")
|
||||
|
||||
et.Tick(103)
|
||||
require.Equal(t, 0, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls")
|
||||
|
||||
var h uint32
|
||||
for h = 104; h < 109; h++ {
|
||||
et.Tick(h)
|
||||
require.Equal(t, 0, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls")
|
||||
}
|
||||
|
||||
et.Tick(109)
|
||||
require.Equal(t, 1, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls")
|
||||
|
||||
et.Tick(110)
|
||||
require.Equal(t, 1, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls")
|
||||
|
||||
et.Tick(111)
|
||||
require.Equal(t, 1, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls")
|
||||
|
||||
et.Tick(112)
|
||||
require.Equal(t, 1, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls")
|
||||
|
||||
et.Tick(113)
|
||||
require.Equal(t, 1, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls")
|
||||
|
||||
for h = 114; h < 119; h++ {
|
||||
et.Tick(h)
|
||||
require.Equal(t, 1, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls")
|
||||
}
|
||||
et.Tick(120)
|
||||
require.Equal(t, 2, neh.called, "invalid new epoch handler calls")
|
||||
require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls")
|
||||
}
|
||||
|
||||
type testAlphabetState struct {
|
||||
isAlphabet bool
|
||||
}
|
||||
|
||||
func (s *testAlphabetState) IsAlphabet() bool {
|
||||
return s.isAlphabet
|
||||
}
|
||||
|
||||
type testNewEpochHandler struct {
|
||||
|
@ -100,15 +71,6 @@ func (h *testNewEpochHandler) Handle() {
|
|||
h.called++
|
||||
}
|
||||
|
||||
type testContainerEstStopper struct {
|
||||
called int
|
||||
}
|
||||
|
||||
func (s *testContainerEstStopper) StopEstimation(_ container.StopEstimationPrm) error {
|
||||
s.called++
|
||||
return nil
|
||||
}
|
||||
|
||||
type testEpochState struct {
|
||||
counter uint64
|
||||
duration uint64
|
||||
|
|
|
@ -36,7 +36,6 @@ import (
|
|||
)
|
||||
|
||||
func (s *Server) initNetmapProcessor(cfg *viper.Viper,
|
||||
cnrClient *container.Client,
|
||||
alphaSync event.Handler,
|
||||
) error {
|
||||
locodeValidator, err := s.newLocodeValidator(cfg)
|
||||
|
@ -59,7 +58,6 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper,
|
|||
AlphabetState: s,
|
||||
CleanupEnabled: cfg.GetBool("netmap_cleaner.enabled"),
|
||||
CleanupThreshold: cfg.GetUint64("netmap_cleaner.threshold"),
|
||||
ContainerWrapper: cnrClient,
|
||||
NotaryDepositHandler: s.onlyAlphabetEventHandler(
|
||||
s.notaryHandler,
|
||||
),
|
||||
|
@ -198,15 +196,10 @@ func (s *Server) createIRFetcher() irFetcher {
|
|||
return irf
|
||||
}
|
||||
|
||||
func (s *Server) initTimers(cfg *viper.Viper, morphClients *serverMorphClients) {
|
||||
func (s *Server) initTimers(cfg *viper.Viper) {
|
||||
s.epochTimer = newEpochTimer(&epochTimerArgs{
|
||||
l: s.log,
|
||||
alphabetState: s,
|
||||
newEpochHandlers: s.newEpochTickHandlers(),
|
||||
cnrWrapper: morphClients.CnrClient,
|
||||
epoch: s,
|
||||
stopEstimationDMul: cfg.GetUint32("timers.stop_estimation.mul"),
|
||||
stopEstimationDDiv: cfg.GetUint32("timers.stop_estimation.div"),
|
||||
newEpochHandlers: s.newEpochTickHandlers(),
|
||||
epoch: s,
|
||||
})
|
||||
|
||||
s.addBlockTimer(s.epochTimer)
|
||||
|
@ -425,7 +418,7 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien
|
|||
return err
|
||||
}
|
||||
|
||||
err = s.initNetmapProcessor(cfg, morphClients.CnrClient, alphaSync)
|
||||
err = s.initNetmapProcessor(cfg, alphaSync)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -405,7 +405,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan
|
|||
return nil, err
|
||||
}
|
||||
|
||||
server.initTimers(cfg, morphClients)
|
||||
server.initTimers(cfg)
|
||||
|
||||
err = server.initGRPCServer(cfg, log, audit)
|
||||
if err != nil {
|
||||
|
|
|
@ -40,13 +40,14 @@ func (s *Server) depositMainNotary() (tx util.Uint256, err error) {
|
|||
)
|
||||
}
|
||||
|
||||
func (s *Server) depositSideNotary() (tx util.Uint256, err error) {
|
||||
func (s *Server) depositSideNotary() (util.Uint256, error) {
|
||||
depositAmount, err := client.CalculateNotaryDepositAmount(s.morphClient, gasMultiplier, gasDivisor)
|
||||
if err != nil {
|
||||
return util.Uint256{}, fmt.Errorf("could not calculate side notary deposit amount: %w", err)
|
||||
}
|
||||
|
||||
return s.morphClient.DepositEndlessNotary(depositAmount)
|
||||
tx, _, err := s.morphClient.DepositEndlessNotary(depositAmount)
|
||||
return tx, err
|
||||
}
|
||||
|
||||
func (s *Server) notaryHandler(_ event.Event) {
|
||||
|
|
|
@ -5,15 +5,14 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
v2netmap "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
|
||||
netmapContract "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance"
|
||||
timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers"
|
||||
cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
|
||||
netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event"
|
||||
netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
|
||||
v2netmap "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"github.com/nspcc-dev/neo-go/pkg/core/transaction"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
|
@ -68,7 +67,6 @@ func TestNewEpoch(t *testing.T) {
|
|||
duration: 10,
|
||||
}
|
||||
r := &testEpochResetter{}
|
||||
cc := &testContainerClient{}
|
||||
nc := &testNetmapClient{
|
||||
epochDuration: 20,
|
||||
txHeights: map[util.Uint256]uint32{
|
||||
|
@ -82,7 +80,6 @@ func TestNewEpoch(t *testing.T) {
|
|||
p.NotaryDepositHandler = eh.Handle
|
||||
p.AlphabetSyncHandler = eh.Handle
|
||||
p.NetmapClient = nc
|
||||
p.ContainerWrapper = cc
|
||||
p.EpochTimer = r
|
||||
p.EpochState = es
|
||||
})
|
||||
|
@ -103,11 +100,6 @@ func TestNewEpoch(t *testing.T) {
|
|||
require.Equal(t, ev.Num, es.counter, "invalid epoch counter")
|
||||
require.EqualValues(t, []uint32{nc.txHeights[ev.Hash]}, r.timers, "invalid epoch timer resets")
|
||||
|
||||
var expEstimation cntClient.StartEstimationPrm
|
||||
expEstimation.SetEpoch(ev.Num - 1)
|
||||
expEstimation.SetHash(ev.Hash)
|
||||
require.EqualValues(t, []cntClient.StartEstimationPrm{expEstimation}, cc.estimations, "invalid estimations")
|
||||
|
||||
require.EqualValues(t, []event.Event{
|
||||
governance.NewSyncEvent(ev.TxHash()),
|
||||
ev,
|
||||
|
@ -274,7 +266,6 @@ func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) {
|
|||
as := &testAlphabetState{
|
||||
isAlphabet: true,
|
||||
}
|
||||
cc := &testContainerClient{}
|
||||
nc := &testNetmapClient{}
|
||||
eh := &testEventHandler{}
|
||||
|
||||
|
@ -288,7 +279,6 @@ func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) {
|
|||
EpochState: es,
|
||||
EpochTimer: r,
|
||||
AlphabetState: as,
|
||||
ContainerWrapper: cc,
|
||||
NetmapClient: nc,
|
||||
NotaryDepositHandler: eh.Handle,
|
||||
AlphabetSyncHandler: eh.Handle,
|
||||
|
@ -354,15 +344,6 @@ func (s *testAlphabetState) IsAlphabet() bool {
|
|||
return s.isAlphabet
|
||||
}
|
||||
|
||||
type testContainerClient struct {
|
||||
estimations []cntClient.StartEstimationPrm
|
||||
}
|
||||
|
||||
func (c *testContainerClient) StartEstimation(p cntClient.StartEstimationPrm) error {
|
||||
c.estimations = append(c.estimations, p)
|
||||
return nil
|
||||
}
|
||||
|
||||
type notaryInvoke struct {
|
||||
contract util.Uint160
|
||||
fee fixedn.Fixed8
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
package netmap
|
||||
|
||||
import (
|
||||
v2netmap "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||
v2netmap "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
|
|
@ -3,7 +3,6 @@ package netmap
|
|||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance"
|
||||
cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
|
||||
netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -44,20 +43,6 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
prm := cntClient.StartEstimationPrm{}
|
||||
|
||||
prm.SetEpoch(epoch - 1)
|
||||
prm.SetHash(ev.TxHash())
|
||||
|
||||
if epoch > 0 && np.alphabetState.IsAlphabet() { // estimates are invalid in genesis epoch
|
||||
err = np.containerWrp.StartEstimation(prm)
|
||||
if err != nil {
|
||||
np.log.Warn(logs.NetmapCantStartContainerSizeEstimation,
|
||||
zap.Uint64("epoch", epoch),
|
||||
zap.String("error", err.Error()))
|
||||
}
|
||||
}
|
||||
|
||||
np.netmapSnapshot.update(*networkMap, epoch)
|
||||
np.handleCleanupTick(netmapCleanupTick{epoch: epoch, txHash: ev.TxHash()})
|
||||
np.handleAlphabetSync(governance.NewSyncEvent(ev.TxHash()))
|
||||
|
|
|
@ -7,7 +7,6 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state"
|
||||
cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event"
|
||||
netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
|
||||
|
@ -65,10 +64,6 @@ type (
|
|||
MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error
|
||||
}
|
||||
|
||||
ContainerClient interface {
|
||||
StartEstimation(p cntClient.StartEstimationPrm) error
|
||||
}
|
||||
|
||||
// Processor of events produced by network map contract
|
||||
// and new epoch ticker, because it is related to contract.
|
||||
Processor struct {
|
||||
|
@ -80,7 +75,6 @@ type (
|
|||
alphabetState AlphabetState
|
||||
|
||||
netmapClient Client
|
||||
containerWrp ContainerClient
|
||||
|
||||
netmapSnapshot cleanupTable
|
||||
|
||||
|
@ -103,7 +97,6 @@ type (
|
|||
AlphabetState AlphabetState
|
||||
CleanupEnabled bool
|
||||
CleanupThreshold uint64 // in epochs
|
||||
ContainerWrapper ContainerClient
|
||||
|
||||
AlphabetSyncHandler event.Handler
|
||||
NotaryDepositHandler event.Handler
|
||||
|
@ -133,8 +126,6 @@ func New(p *Params) (*Processor, error) {
|
|||
return nil, errors.New("ir/netmap: alphabet sync handler is not set")
|
||||
case p.NotaryDepositHandler == nil:
|
||||
return nil, errors.New("ir/netmap: notary deposit handler is not set")
|
||||
case p.ContainerWrapper == nil:
|
||||
return nil, errors.New("ir/netmap: container contract wrapper is not set")
|
||||
case p.NodeValidator == nil:
|
||||
return nil, errors.New("ir/netmap: node validator is not set")
|
||||
case p.NodeStateSettings == nil:
|
||||
|
@ -161,7 +152,6 @@ func New(p *Params) (*Processor, error) {
|
|||
epochState: p.EpochState,
|
||||
alphabetState: p.AlphabetState,
|
||||
netmapClient: p.NetmapClient,
|
||||
containerWrp: p.ContainerWrapper,
|
||||
netmapSnapshot: newCleanupTable(p.CleanupEnabled, p.CleanupThreshold),
|
||||
|
||||
handleAlphabetSync: p.AlphabetSyncHandler,
|
||||
|
|
|
@ -61,7 +61,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) {
|
|||
require.NoError(t, b.Init())
|
||||
|
||||
storageIDs := make(map[oid.Address][]byte)
|
||||
for i := 0; i < 100; i++ {
|
||||
for range 100 {
|
||||
obj := blobstortest.NewObject(64 * 1024) // 64KB object
|
||||
data, err := obj.Marshal()
|
||||
require.NoError(t, err)
|
||||
|
@ -168,7 +168,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) {
|
|||
|
||||
storageIDs := make(map[oid.Address][]byte)
|
||||
toDelete := make(map[oid.Address][]byte)
|
||||
for i := 0; i < 100; i++ { // 2 objects for one blobovnicza, so 50 DBs total will be created
|
||||
for i := range 100 { // 2 objects for one blobovnicza, so 50 DBs total will be created
|
||||
obj := blobstortest.NewObject(64 * 1024)
|
||||
data, err := obj.Marshal()
|
||||
require.NoError(t, err)
|
||||
|
@ -236,7 +236,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) {
|
|||
require.NoError(t, b.Init())
|
||||
|
||||
storageIDs := make(map[oid.Address][]byte)
|
||||
for i := 0; i < 100; i++ { // 2 objects for one blobovnicza, so 50 DBs total will be created
|
||||
for range 100 { // 2 objects for one blobovnicza, so 50 DBs total will be created
|
||||
obj := blobstortest.NewObject(64 * 1024)
|
||||
data, err := obj.Marshal()
|
||||
require.NoError(t, err)
|
||||
|
|
|
@ -47,7 +47,7 @@ func TestIterateObjects(t *testing.T) {
|
|||
|
||||
mObjs := make(map[string]addrData)
|
||||
|
||||
for i := uint64(0); i < objNum; i++ {
|
||||
for i := range uint64(objNum) {
|
||||
sz := smalSz
|
||||
|
||||
big := i < objNum/2
|
||||
|
|
|
@ -151,7 +151,7 @@ func TestErrorReporting(t *testing.T) {
|
|||
checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite)
|
||||
}
|
||||
|
||||
for i := uint32(0); i < 2; i++ {
|
||||
for i := range uint32(2) {
|
||||
_, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)})
|
||||
require.Error(t, err)
|
||||
checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly)
|
||||
|
|
|
@ -6,12 +6,12 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
|
||||
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
|
|
|
@ -110,6 +110,34 @@ func (e *StorageEngine) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID str
|
|||
return nil
|
||||
}
|
||||
|
||||
// TreeApply implements the pilorama.Forest interface.
|
||||
func (e *StorageEngine) TreeApplyBatch(ctx context.Context, cnr cidSDK.ID, treeID string, m []*pilorama.Move) error {
|
||||
ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeApplyBatch",
|
||||
trace.WithAttributes(
|
||||
attribute.String("container_id", cnr.EncodeToString()),
|
||||
attribute.String("tree_id", treeID),
|
||||
),
|
||||
)
|
||||
defer span.End()
|
||||
|
||||
index, lst, err := e.getTreeShard(ctx, cnr, treeID)
|
||||
if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) {
|
||||
return err
|
||||
}
|
||||
|
||||
err = lst[index].TreeApplyBatch(ctx, cnr, treeID, m)
|
||||
if err != nil {
|
||||
if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled {
|
||||
e.reportShardError(lst[index], "can't perform `TreeApplyBatch`", err,
|
||||
zap.Stringer("cid", cnr),
|
||||
zap.String("tree", treeID),
|
||||
zap.String("trace_id", tracingPkg.GetTraceID(ctx)))
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TreeGetByPath implements the pilorama.Forest interface.
|
||||
func (e *StorageEngine) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]pilorama.Node, error) {
|
||||
ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetByPath",
|
||||
|
|
|
@ -11,9 +11,9 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
|
||||
v2object "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/mr-tron/base58"
|
||||
"go.etcd.io/bbolt"
|
||||
|
|
|
@ -6,10 +6,10 @@ import (
|
|||
"strconv"
|
||||
"testing"
|
||||
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
|
||||
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
|
|
@ -5,10 +5,10 @@ import (
|
|||
"strconv"
|
||||
"testing"
|
||||
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
object2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
|
||||
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
|
|
@ -9,12 +9,12 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
||||
storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
|
|
@ -8,9 +8,9 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
||||
v2object "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
|
|
@ -7,10 +7,10 @@ import (
|
|||
"strconv"
|
||||
"testing"
|
||||
|
||||
v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
|
||||
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
|
||||
v2object "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
|
|
|
@ -12,8 +12,8 @@ import (
|
|||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"go.etcd.io/bbolt"
|
||||
|
|
|
@ -11,12 +11,12 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
|
||||
objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
package pilorama
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"encoding/binary"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
@ -48,8 +48,8 @@ func (b *batch) run() {
|
|||
|
||||
// Sorting without a mutex is ok, because we append to this slice only if timer is non-nil.
|
||||
// See (*boltForest).addBatch for details.
|
||||
sort.Slice(b.operations, func(i, j int) bool {
|
||||
return b.operations[i].Time < b.operations[j].Time
|
||||
slices.SortFunc(b.operations, func(mi, mj *Move) int {
|
||||
return cmp.Compare(mi.Time, mj.Time)
|
||||
})
|
||||
b.operations = slices.CompactFunc(b.operations, func(x, y *Move) bool { return x.Time == y.Time })
|
||||
|
||||
|
|
|
@ -10,7 +10,6 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
@ -559,6 +558,82 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string
|
|||
return metaerr.Wrap(err)
|
||||
}
|
||||
|
||||
func (t *boltForest) TreeApplyBatch(ctx context.Context, cnr cidSDK.ID, treeID string, m []*Move) error {
|
||||
var (
|
||||
startedAt = time.Now()
|
||||
success = false
|
||||
)
|
||||
defer func() {
|
||||
t.metrics.AddMethodDuration("TreeApplyBatch", time.Since(startedAt), success)
|
||||
}()
|
||||
|
||||
_, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeApplyBatch",
|
||||
trace.WithAttributes(
|
||||
attribute.String("container_id", cnr.EncodeToString()),
|
||||
attribute.String("tree_id", treeID),
|
||||
),
|
||||
)
|
||||
defer span.End()
|
||||
|
||||
m, err := t.filterSeen(cnr, treeID, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(m) == 0 {
|
||||
success = true
|
||||
return nil
|
||||
}
|
||||
|
||||
ch := make(chan error)
|
||||
b := &batch{
|
||||
forest: t,
|
||||
cid: cnr,
|
||||
treeID: treeID,
|
||||
results: []chan<- error{ch},
|
||||
operations: m,
|
||||
}
|
||||
go func() {
|
||||
b.run()
|
||||
}()
|
||||
err = <-ch
|
||||
success = err == nil
|
||||
return metaerr.Wrap(err)
|
||||
}
|
||||
|
||||
func (t *boltForest) filterSeen(cnr cidSDK.ID, treeID string, m []*Move) ([]*Move, error) {
|
||||
t.modeMtx.RLock()
|
||||
defer t.modeMtx.RUnlock()
|
||||
|
||||
if t.mode.NoMetabase() {
|
||||
return nil, ErrDegradedMode
|
||||
}
|
||||
|
||||
ops := make([]*Move, 0, len(m))
|
||||
for _, op := range m {
|
||||
var seen bool
|
||||
err := t.db.View(func(tx *bbolt.Tx) error {
|
||||
treeRoot := tx.Bucket(bucketName(cnr, treeID))
|
||||
if treeRoot == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
b := treeRoot.Bucket(logBucket)
|
||||
|
||||
var logKey [8]byte
|
||||
binary.BigEndian.PutUint64(logKey[:], op.Time)
|
||||
seen = b.Get(logKey[:]) != nil
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, metaerr.Wrap(err)
|
||||
}
|
||||
if !seen {
|
||||
ops = append(ops, op)
|
||||
}
|
||||
}
|
||||
return ops, nil
|
||||
}
|
||||
|
||||
// TreeApplyStream should be used with caution: this method locks other write transactions while `source` is not closed.
|
||||
func (t *boltForest) TreeApplyStream(ctx context.Context, cnr cidSDK.ID, treeID string, source <-chan *Move) error {
|
||||
var (
|
||||
|
@ -705,7 +780,7 @@ func (t *boltForest) applyOperation(logBucket, treeBucket *bbolt.Bucket, ms []*M
|
|||
key, value = c.Prev()
|
||||
}
|
||||
|
||||
for i := range len(ms) {
|
||||
for i := range ms {
|
||||
// Loop invariant: key represents the next stored timestamp after ms[i].Time.
|
||||
|
||||
// 2. Insert the operation.
|
||||
|
@ -1093,14 +1168,19 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr
|
|||
return res, last, metaerr.Wrap(err)
|
||||
}
|
||||
|
||||
func sortByFilename(nodes []NodeInfo) {
|
||||
slices.SortFunc(nodes, func(a, b NodeInfo) int {
|
||||
return bytes.Compare(a.Meta.GetAttr(AttributeFilename), b.Meta.GetAttr(AttributeFilename))
|
||||
})
|
||||
}
|
||||
|
||||
func sortAndCut(result []NodeInfo, last *string) []NodeInfo {
|
||||
var lastBytes []byte
|
||||
if last != nil {
|
||||
lastBytes = []byte(*last)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
return bytes.Compare(result[i].Meta.GetAttr(AttributeFilename), result[j].Meta.GetAttr(AttributeFilename)) == -1
|
||||
})
|
||||
sortByFilename(result)
|
||||
|
||||
for i := range result {
|
||||
if lastBytes == nil || bytes.Compare(lastBytes, result[i].Meta.GetAttr(AttributeFilename)) == -1 {
|
||||
return result[i:]
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue