From 634de975094b8f83b5b9eeff15b7fd2bfdd6f1fc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Apr 2025 10:09:05 +0300 Subject: [PATCH 01/94] [#1704] metabase: Do not ignore errors by Delete Change-Id: Ie7b89071a007f53f55879ff9e7e0c25d24ad5dbf Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete.go | 59 ++++++++++++--------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index d338e228f..cf1d563e9 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -376,11 +376,12 @@ func parentLength(tx *bbolt.Tx, addr oid.Address) int { return len(lst) } -func delUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) { +func delUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error { bkt := tx.Bucket(item.name) if bkt != nil { - _ = bkt.Delete(item.key) // ignore error, best effort there + return bkt.Delete(item.key) } + return nil } func delListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { @@ -405,19 +406,16 @@ func delListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { // if list empty, remove the key from bucket if len(lst) == 0 { - _ = bkt.Delete(item.key) // ignore error, best effort there - - return nil + return bkt.Delete(item.key) } // if list is not empty, then update it encodedLst, err := encodeList(lst) if err != nil { - return nil // ignore error, best effort there + return err } - _ = bkt.Put(item.key, encodedLst) // ignore error, best effort there - return nil + return bkt.Put(item.key, encodedLst) } func delFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { @@ -480,35 +478,47 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error return ErrUnknownObjectType } - delUniqueIndexItem(tx, namedBucketItem{ + if err := delUniqueIndexItem(tx, namedBucketItem{ name: bucketName, key: objKey, - }) + }); err != nil { + return err + } } else { - delUniqueIndexItem(tx, namedBucketItem{ + if err := delUniqueIndexItem(tx, namedBucketItem{ name: parentBucketName(cnr, bucketName), key: objKey, - }) + }); err != nil { + return err + } } - delUniqueIndexItem(tx, namedBucketItem{ // remove from storage id index + if err := delUniqueIndexItem(tx, namedBucketItem{ // remove from storage id index name: smallBucketName(cnr, bucketName), key: objKey, - }) - delUniqueIndexItem(tx, namedBucketItem{ // remove from root index + }); err != nil { + return err + } + if err := delUniqueIndexItem(tx, namedBucketItem{ // remove from root index name: rootBucketName(cnr, bucketName), key: objKey, - }) + }); err != nil { + return err + } if expEpoch, ok := hasExpirationEpoch(obj); ok { - delUniqueIndexItem(tx, namedBucketItem{ + if err := delUniqueIndexItem(tx, namedBucketItem{ name: expEpochToObjectBucketName, key: expirationEpochKey(expEpoch, cnr, addr.Object()), - }) - delUniqueIndexItem(tx, namedBucketItem{ + }); err != nil { + return err + } + if err := delUniqueIndexItem(tx, namedBucketItem{ name: objectToExpirationEpochBucketName(cnr, make([]byte, bucketKeySize)), key: objKey, - }) + }); err != nil { + return err + } } return nil @@ -535,10 +545,12 @@ func deleteECRelatedInfo(tx *bbolt.Tx, garbageBKT *bbolt.Bucket, obj *objectSDK. // also drop EC parent root info if current EC chunk is the last one if !hasAnyChunks { - delUniqueIndexItem(tx, namedBucketItem{ + if err := delUniqueIndexItem(tx, namedBucketItem{ name: rootBucketName(cnr, make([]byte, bucketKeySize)), key: objectKey(ech.Parent(), make([]byte, objectKeySize)), - }) + }); err != nil { + return err + } } if ech.ParentSplitParentID() == nil { @@ -572,11 +584,10 @@ func deleteECRelatedInfo(tx *bbolt.Tx, garbageBKT *bbolt.Bucket, obj *objectSDK. } // drop split info - delUniqueIndexItem(tx, namedBucketItem{ + return delUniqueIndexItem(tx, namedBucketItem{ name: rootBucketName(cnr, make([]byte, bucketKeySize)), key: objectKey(*ech.ParentSplitParentID(), make([]byte, objectKeySize)), }) - return nil } func hasAnyECChunks(tx *bbolt.Tx, ech *objectSDK.ECHeader, cnr cid.ID) bool { From 50dccff7c1e16eb242bfe192ddc3b93d91c03427 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 10 Mar 2025 11:43:40 +0300 Subject: [PATCH 02/94] [#1633] morph/netmap: Merge node info in netmap with candidates list Applicable for both cases: when node uses local cache for netmap and when it disabled. Change-Id: I3050f537e20312a4b39e944aca763b77bd1e74c4 Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/cache.go | 292 +++++++++++++++++++----- cmd/frostfs-node/cache_test.go | 74 ++++++ cmd/frostfs-node/config/morph/config.go | 17 ++ cmd/frostfs-node/morph.go | 5 +- cmd/frostfs-node/netmap_source.go | 55 +++++ config/example/node.yaml | 3 + docs/storage-node-configuration.md | 18 +- internal/logs/logs.go | 1 + 8 files changed, 397 insertions(+), 68 deletions(-) create mode 100644 cmd/frostfs-node/netmap_source.go diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 0fe56d2b0..38cee5837 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -1,20 +1,27 @@ package main import ( + "bytes" + "cmp" "context" + "slices" "sync" + "sync/atomic" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - lru "github.com/hashicorp/golang-lru/v2" "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/hashicorp/golang-lru/v2/simplelru" + "go.uber.org/zap" ) type netValueReader[K any, V any] func(ctx context.Context, cid K) (V, error) @@ -110,55 +117,6 @@ func (c *ttlNetCache[K, V]) remove(key K) { hit = c.cache.Remove(key) } -// entity that provides LRU cache interface. -type lruNetCache struct { - cache *lru.Cache[uint64, *netmapSDK.NetMap] - - netRdr netValueReader[uint64, *netmapSDK.NetMap] - - metrics cacheMetrics -} - -// newNetworkLRUCache returns wrapper over netValueReader with LRU cache. -func newNetworkLRUCache(sz int, netRdr netValueReader[uint64, *netmapSDK.NetMap], metrics cacheMetrics) *lruNetCache { - cache, err := lru.New[uint64, *netmapSDK.NetMap](sz) - fatalOnErr(err) - - return &lruNetCache{ - cache: cache, - netRdr: netRdr, - metrics: metrics, - } -} - -// reads value by the key. -// -// updates the value from the network on cache miss. -// -// returned value should not be modified. -func (c *lruNetCache) get(ctx context.Context, key uint64) (*netmapSDK.NetMap, error) { - hit := false - startedAt := time.Now() - defer func() { - c.metrics.AddMethodDuration("Get", time.Since(startedAt), hit) - }() - - val, ok := c.cache.Get(key) - if ok { - hit = true - return val, nil - } - - val, err := c.netRdr(ctx, key) - if err != nil { - return nil, err - } - - c.cache.Add(key, val) - - return val, nil -} - // wrapper over TTL cache of values read from the network // that implements container storage. type ttlContainerStorage struct { @@ -200,20 +158,236 @@ func (s ttlContainerStorage) DeletionInfo(ctx context.Context, cnr cid.ID) (*con type lruNetmapSource struct { netState netmap.State - cache *lruNetCache + client rawSource + cache *simplelru.LRU[uint64, *atomic.Pointer[netmapSDK.NetMap]] + mtx sync.RWMutex + metrics cacheMetrics + log *logger.Logger + candidates atomic.Pointer[[]netmapSDK.NodeInfo] } -func newCachedNetmapStorage(s netmap.State, v netmap.Source) netmap.Source { +type rawSource interface { + GetCandidates(ctx context.Context) ([]netmapSDK.NodeInfo, error) + GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) +} + +func newCachedNetmapStorage(ctx context.Context, log *logger.Logger, + netState netmap.State, client rawSource, wg *sync.WaitGroup, d time.Duration, +) netmap.Source { const netmapCacheSize = 10 - lruNetmapCache := newNetworkLRUCache(netmapCacheSize, func(ctx context.Context, key uint64) (*netmapSDK.NetMap, error) { - return v.GetNetMapByEpoch(ctx, key) - }, metrics.NewCacheMetrics("netmap")) + cache, err := simplelru.NewLRU[uint64, *atomic.Pointer[netmapSDK.NetMap]](netmapCacheSize, nil) + fatalOnErr(err) - return &lruNetmapSource{ - netState: s, - cache: lruNetmapCache, + src := &lruNetmapSource{ + netState: netState, + client: client, + cache: cache, + log: log, + metrics: metrics.NewCacheMetrics("netmap"), } + + wg.Add(1) + go func() { + defer wg.Done() + src.updateCandidates(ctx, d) + }() + + return src +} + +// updateCandidates routine to merge netmap in cache with candidates list. +func (s *lruNetmapSource) updateCandidates(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + newCandidates, err := s.client.GetCandidates(ctx) + if err != nil { + s.log.Debug(ctx, logs.FailedToUpdateNetmapCandidates, zap.Error(err)) + timer.Reset(d) + break + } + if len(newCandidates) == 0 { + s.candidates.Store(&newCandidates) + timer.Reset(d) + break + } + slices.SortFunc(newCandidates, func(n1 netmapSDK.NodeInfo, n2 netmapSDK.NodeInfo) int { + return cmp.Compare(n1.Hash(), n2.Hash()) + }) + + // Check once state changed + v := s.candidates.Load() + if v == nil { + s.candidates.Store(&newCandidates) + s.mergeCacheWithCandidates(newCandidates) + timer.Reset(d) + break + } + ret := slices.CompareFunc(*v, newCandidates, func(n1 netmapSDK.NodeInfo, n2 netmapSDK.NodeInfo) int { + if !bytes.Equal(n1.PublicKey(), n2.PublicKey()) || + uint32(n1.Status()) != uint32(n2.Status()) || + slices.Compare(n1.ExternalAddresses(), n2.ExternalAddresses()) != 0 { + return 1 + } + var ne1 []string + n1.IterateNetworkEndpoints(func(s string) bool { + ne1 = append(ne1, s) + return false + }) + var ne2 []string + n2.IterateNetworkEndpoints(func(s string) bool { + ne2 = append(ne2, s) + return false + }) + return slices.Compare(ne1, ne2) + }) + if ret != 0 { + s.candidates.Store(&newCandidates) + s.mergeCacheWithCandidates(newCandidates) + } + timer.Reset(d) + } + } +} + +func (s *lruNetmapSource) mergeCacheWithCandidates(candidates []netmapSDK.NodeInfo) { + s.mtx.Lock() + tmp := s.cache.Values() + s.mtx.Unlock() + for _, pointer := range tmp { + nm := pointer.Load() + updates := getNetMapNodesToUpdate(nm, candidates) + if len(updates) > 0 { + nm = nm.Clone() + mergeNetmapWithCandidates(updates, nm) + pointer.Store(nm) + } + } +} + +// reads value by the key. +// +// updates the value from the network on cache miss. +// +// returned value should not be modified. +func (s *lruNetmapSource) get(ctx context.Context, key uint64) (*netmapSDK.NetMap, error) { + hit := false + startedAt := time.Now() + defer func() { + s.metrics.AddMethodDuration("Get", time.Since(startedAt), hit) + }() + + s.mtx.RLock() + val, ok := s.cache.Get(key) + s.mtx.RUnlock() + if ok { + hit = true + return val.Load(), nil + } + + s.mtx.Lock() + defer s.mtx.Unlock() + + val, ok = s.cache.Get(key) + if ok { + hit = true + return val.Load(), nil + } + + nm, err := s.client.GetNetMapByEpoch(ctx, key) + if err != nil { + return nil, err + } + v := s.candidates.Load() + if v != nil { + updates := getNetMapNodesToUpdate(nm, *v) + if len(updates) > 0 { + mergeNetmapWithCandidates(updates, nm) + } + } + + p := atomic.Pointer[netmapSDK.NetMap]{} + p.Store(nm) + s.cache.Add(key, &p) + + return nm, nil +} + +// mergeNetmapWithCandidates updates nodes state in the provided netmap with state in the list of candidates. +func mergeNetmapWithCandidates(updates []nodeToUpdate, nm *netmapSDK.NetMap) { + for _, v := range updates { + if v.status != netmapSDK.UnspecifiedState { + nm.Nodes()[v.netmapIndex].SetStatus(v.status) + } + if v.externalAddresses != nil { + nm.Nodes()[v.netmapIndex].SetExternalAddresses(v.externalAddresses...) + } + if v.endpoints != nil { + nm.Nodes()[v.netmapIndex].SetNetworkEndpoints(v.endpoints...) + } + } +} + +type nodeToUpdate struct { + netmapIndex int + status netmapSDK.NodeState + externalAddresses []string + endpoints []string +} + +// getNetMapNodesToUpdate checks for the changes between provided netmap and the list of candidates. +func getNetMapNodesToUpdate(nm *netmapSDK.NetMap, candidates []netmapSDK.NodeInfo) []nodeToUpdate { + var res []nodeToUpdate + for i := range nm.Nodes() { + for _, cnd := range candidates { + if bytes.Equal(nm.Nodes()[i].PublicKey(), cnd.PublicKey()) { + var tmp nodeToUpdate + var update bool + + if cnd.Status() != nm.Nodes()[i].Status() && + (cnd.Status() == netmapSDK.Online || cnd.Status() == netmapSDK.Maintenance) { + update = true + tmp.status = cnd.Status() + } + + externalAddresses := cnd.ExternalAddresses() + if externalAddresses != nil && + slices.Compare(externalAddresses, nm.Nodes()[i].ExternalAddresses()) != 0 { + update = true + tmp.externalAddresses = externalAddresses + } + + nodeEndpoints := make([]string, 0, nm.Nodes()[i].NumberOfNetworkEndpoints()) + nm.Nodes()[i].IterateNetworkEndpoints(func(s string) bool { + nodeEndpoints = append(nodeEndpoints, s) + return false + }) + candidateEndpoints := make([]string, 0, cnd.NumberOfNetworkEndpoints()) + cnd.IterateNetworkEndpoints(func(s string) bool { + candidateEndpoints = append(candidateEndpoints, s) + return false + }) + if slices.Compare(nodeEndpoints, candidateEndpoints) != 0 { + update = true + tmp.endpoints = candidateEndpoints + } + + if update { + tmp.netmapIndex = i + res = append(res, tmp) + } + + break + } + } + } + return res } func (s *lruNetmapSource) GetNetMap(ctx context.Context, diff uint64) (*netmapSDK.NetMap, error) { @@ -225,7 +399,7 @@ func (s *lruNetmapSource) GetNetMapByEpoch(ctx context.Context, epoch uint64) (* } func (s *lruNetmapSource) getNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) { - val, err := s.cache.get(ctx, epoch) + val, err := s.get(ctx, epoch) if err != nil { return nil, err } diff --git a/cmd/frostfs-node/cache_test.go b/cmd/frostfs-node/cache_test.go index b1601aa67..24286826f 100644 --- a/cmd/frostfs-node/cache_test.go +++ b/cmd/frostfs-node/cache_test.go @@ -3,9 +3,11 @@ package main import ( "context" "errors" + "sync" "testing" "time" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/stretchr/testify/require" ) @@ -59,3 +61,75 @@ func testNetValueReader(_ context.Context, key string) (time.Time, error) { type noopCacheMetricts struct{} func (m *noopCacheMetricts) AddMethodDuration(method string, d time.Duration, hit bool) {} + +type rawSrc struct{} + +func (r *rawSrc) GetCandidates(_ context.Context) ([]netmapSDK.NodeInfo, error) { + node0 := netmapSDK.NodeInfo{} + node0.SetPublicKey([]byte{byte(1)}) + node0.SetStatus(netmapSDK.Online) + node0.SetExternalAddresses("1", "0") + node0.SetNetworkEndpoints("1", "0") + + node1 := netmapSDK.NodeInfo{} + node1.SetPublicKey([]byte{byte(1)}) + node1.SetStatus(netmapSDK.Online) + node1.SetExternalAddresses("1", "0") + node1.SetNetworkEndpoints("1", "0") + + return []netmapSDK.NodeInfo{node0, node1}, nil +} + +func (r *rawSrc) GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) { + nm := netmapSDK.NetMap{} + nm.SetEpoch(1) + + node0 := netmapSDK.NodeInfo{} + node0.SetPublicKey([]byte{byte(1)}) + node0.SetStatus(netmapSDK.Maintenance) + node0.SetExternalAddresses("0") + node0.SetNetworkEndpoints("0") + + node1 := netmapSDK.NodeInfo{} + node1.SetPublicKey([]byte{byte(1)}) + node1.SetStatus(netmapSDK.Maintenance) + node1.SetExternalAddresses("0") + node1.SetNetworkEndpoints("0") + + nm.SetNodes([]netmapSDK.NodeInfo{node0, node1}) + + return &nm, nil +} + +type st struct{} + +func (s *st) CurrentEpoch() uint64 { + return 1 +} + +func TestNetmapStorage(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + wg := sync.WaitGroup{} + cache := newCachedNetmapStorage(ctx, nil, &st{}, &rawSrc{}, &wg, time.Millisecond*50) + + nm, err := cache.GetNetMapByEpoch(ctx, 1) + require.NoError(t, err) + require.True(t, nm.Nodes()[0].Status() == netmapSDK.Maintenance) + require.True(t, len(nm.Nodes()[0].ExternalAddresses()) == 1) + require.True(t, nm.Nodes()[0].NumberOfNetworkEndpoints() == 1) + + require.Eventually(t, func() bool { + nm, err := cache.GetNetMapByEpoch(ctx, 1) + require.NoError(t, err) + for _, node := range nm.Nodes() { + if !(node.Status() == netmapSDK.Online && len(node.ExternalAddresses()) == 2 && + node.NumberOfNetworkEndpoints() == 2) { + return false + } + } + return true + }, time.Second*5, time.Millisecond*10) + + cancel() + wg.Wait() +} diff --git a/cmd/frostfs-node/config/morph/config.go b/cmd/frostfs-node/config/morph/config.go index d089870ea..a9f774d18 100644 --- a/cmd/frostfs-node/config/morph/config.go +++ b/cmd/frostfs-node/config/morph/config.go @@ -33,6 +33,9 @@ const ( // ContainerCacheSizeDefault represents the default size for the container cache. ContainerCacheSizeDefault = 100 + + // PollCandidatesTimeoutDefault is a default poll timeout for netmap candidates. + PollCandidatesTimeoutDefault = 20 * time.Second ) var errNoMorphEndpoints = errors.New("no morph chain RPC endpoints, see `morph.rpc_endpoint` section") @@ -154,3 +157,17 @@ func FrostfsIDCacheSize(c *config.Config) uint32 { } return config.Uint32Safe(c.Sub(subsection), "frostfsid_cache_size") } + +// NetmapCandidatesPollInterval returns the value of "netmap.candidates.poll_interval" config parameter +// from "morph" section. +// +// Returns PollCandidatesTimeoutDefault if the value is not positive duration. +func NetmapCandidatesPollInterval(c *config.Config) time.Duration { + v := config.DurationSafe(c.Sub(subsection). + Sub("netmap").Sub("candidates"), "poll_interval") + if v > 0 { + return v + } + + return PollCandidatesTimeoutDefault +} diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 657e22389..d3c0f7b81 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -60,10 +60,11 @@ func (c *cfg) initMorphComponents(ctx context.Context) { } if c.cfgMorph.cacheTTL < 0 { - netmapSource = wrap + netmapSource = newRawNetmapStorage(wrap) } else { // use RPC node as source of netmap (with caching) - netmapSource = newCachedNetmapStorage(c.cfgNetmap.state, wrap) + netmapSource = newCachedNetmapStorage(ctx, c.log, c.cfgNetmap.state, wrap, &c.wg, + morphconfig.NetmapCandidatesPollInterval(c.appCfg)) } c.netMapSource = netmapSource diff --git a/cmd/frostfs-node/netmap_source.go b/cmd/frostfs-node/netmap_source.go new file mode 100644 index 000000000..e6be9cdf5 --- /dev/null +++ b/cmd/frostfs-node/netmap_source.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + netmapClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" +) + +type rawNetmapSource struct { + client *netmapClient.Client +} + +func newRawNetmapStorage(client *netmapClient.Client) netmap.Source { + return &rawNetmapSource{ + client: client, + } +} + +func (s *rawNetmapSource) GetNetMap(ctx context.Context, diff uint64) (*netmapSDK.NetMap, error) { + nm, err := s.client.GetNetMap(ctx, diff) + if err != nil { + return nil, err + } + candidates, err := s.client.GetCandidates(ctx) + if err != nil { + return nil, err + } + updates := getNetMapNodesToUpdate(nm, candidates) + if len(updates) > 0 { + mergeNetmapWithCandidates(updates, nm) + } + return nm, nil +} + +func (s *rawNetmapSource) GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) { + nm, err := s.client.GetNetMapByEpoch(ctx, epoch) + if err != nil { + return nil, err + } + candidates, err := s.client.GetCandidates(ctx) + if err != nil { + return nil, err + } + updates := getNetMapNodesToUpdate(nm, candidates) + if len(updates) > 0 { + mergeNetmapWithCandidates(updates, nm) + } + return nm, nil +} + +func (s *rawNetmapSource) Epoch(ctx context.Context) (uint64, error) { + return s.client.Epoch(ctx) +} diff --git a/config/example/node.yaml b/config/example/node.yaml index a448ba7ce..0b6c7b12c 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -95,6 +95,9 @@ morph: - address: wss://rpc2.morph.frostfs.info:40341/ws priority: 2 ape_chain_cache_size: 100000 + netmap: + candidates: + poll_interval: 20s apiclient: dial_timeout: 15s # timeout for FrostFS API client connection diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index b5c8aadfe..5fe011ece 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -148,15 +148,19 @@ morph: - address: wss://rpc2.morph.frostfs.info:40341/ws priority: 2 switch_interval: 2m + netmap: + candidates: + poll_interval: 20s ``` -| Parameter | Type | Default value | Description | -| ---------------------- | --------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `dial_timeout` | `duration` | `5s` | Timeout for dialing connections to N3 RPCs. | -| `cache_ttl` | `duration` | Morph block time | Sidechain cache TTL value (min interval between similar calls).
Negative value disables caching.
Cached entities: containers, container lists, eACL tables. | -| `rpc_endpoint` | list of [endpoint descriptions](#rpc_endpoint-subsection) | | Array of endpoint descriptions. | -| `switch_interval` | `duration` | `2m` | Time interval between the attempts to connect to the highest priority RPC node if the connection is not established yet. | -| `ape_chain_cache_size` | `int` | `10000` | Size of the morph cache for APE chains. | +| Parameter | Type | Default value | Description | +|-----------------------------------|-----------------------------------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `dial_timeout` | `duration` | `5s` | Timeout for dialing connections to N3 RPCs. | +| `cache_ttl` | `duration` | Morph block time | Sidechain cache TTL value (min interval between similar calls).
Negative value disables caching.
Cached entities: containers, container lists, eACL tables. | +| `rpc_endpoint` | list of [endpoint descriptions](#rpc_endpoint-subsection) | | Array of endpoint descriptions. | +| `switch_interval` | `duration` | `2m` | Time interval between the attempts to connect to the highest priority RPC node if the connection is not established yet. | +| `ape_chain_cache_size` | `int` | `10000` | Size of the morph cache for APE chains. | +| `netmap.candidates.poll_interval` | `duration` | `20s` | Timeout to set up frequency of merge candidates to netmap with netmap in local cache. | ## `rpc_endpoint` subsection | Parameter | Type | Default value | Description | diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 5b42b25ba..3a3ceb150 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -515,4 +515,5 @@ const ( FailedToGetNetmapToAdjustIOTag = "failed to get netmap to adjust IO tag" FailedToValidateIncomingIOTag = "failed to validate incoming IO tag, replaced with `client`" WriteCacheFailedToAcquireRPSQuota = "writecache failed to acquire RPS quota to flush object" + FailedToUpdateNetmapCandidates = "update netmap candidates failed" ) From 272128e61fd20c53aa0a3b02da8bdc77f7779e26 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 13:32:11 +0300 Subject: [PATCH 03/94] Revert "[#652] adm: Group independent stages in batches" This reverts commit d00c606feed8ad776fe6df65b601b81790e7dfbe. There are internal dependencies inside the last stage: first, we register NNS root, only then register add records. Revert for now, will revert back after more testing. Change-Id: I760632b5628caf04849d4a64c714cf286051f357 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/helper/n3client.go | 4 +--- .../internal/modules/morph/initialize/initialize.go | 8 +------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index d6ca012ce..3f3a66cb6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -40,8 +40,6 @@ type ClientContext struct { CommitteeAct *actor.Actor // committee actor with the Global witness scope ReadOnlyInvoker *invoker.Invoker // R/O contract invoker, does not contain any signer SentTxs []HashVUBPair - - AwaitDisabled bool } func NewRemoteClient(v *viper.Viper) (Client, error) { @@ -122,7 +120,7 @@ func (c *ClientContext) SendTx(tx *transaction.Transaction, cmd *cobra.Command, } func (c *ClientContext) AwaitTx(cmd *cobra.Command) error { - if len(c.SentTxs) == 0 || c.AwaitDisabled { + if len(c.SentTxs) == 0 { return nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go index 4d39dc662..cdaf7d3bc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go @@ -39,7 +39,6 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { return err } - initCtx.AwaitDisabled = true cmd.Println("Stage 4.1: Transfer GAS to proxy contract.") if err := transferGASToProxy(initCtx); err != nil { return err @@ -56,10 +55,5 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { } cmd.Println("Stage 7: set addresses in NNS.") - if err := setNNS(initCtx); err != nil { - return err - } - - initCtx.AwaitDisabled = false - return initCtx.AwaitTx() + return setNNS(initCtx) } From 2938498b52369bc3abaac3ab1f8e790bba414e8e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 15:23:41 +0300 Subject: [PATCH 04/94] [#1689] adm: Fix NNS root availability check After https://git.frostfs.info/TrueCloudLab/frostfs-contract/pulls/117 we allow checking for root domain availability directly. Before this commit, NNSRootRegistered() has always returned true, so the actual root registration happened as a side-effect of the following code, because NNS registers all parent domains, if they are missing. Change-Id: Icf98f130e77d31b4af7b69697989183c1c8f6a56 Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/helper/initialize_ctx.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go index 8e5615baa..27052697f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go @@ -21,6 +21,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/smartcontract/context" "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" @@ -28,7 +29,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" - "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -524,12 +524,8 @@ func (c *InitializeContext) NNSRegisterDomainScript(nnsHash, expectedHash util.U } func (c *InitializeContext) NNSRootRegistered(nnsHash util.Uint160, zone string) (bool, error) { - res, err := c.CommitteeAct.Call(nnsHash, "isAvailable", "name."+zone) - if err != nil { - return false, err - } - - return res.State == vmstate.Halt.String(), nil + avail, err := unwrap.Bool(c.CommitteeAct.Call(nnsHash, "isAvailable", zone)) + return !avail, err } func (c *InitializeContext) IsUpdated(ctrHash util.Uint160, cs *ContractState) bool { From 5350632e01f2b01f8068dbfec2b9829ae8721519 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 4 Apr 2025 12:48:22 +0300 Subject: [PATCH 05/94] [#1705] engine/test: Increase evacuation timeout This test was flaky in CI probably because of runner load fluctuations. Let's increase the timeout and see if the flakiness goes away. (close #1705) Change-Id: I76f96e3d6f4adb3d5de0e27b8ee6b47685236277 Signed-off-by: Vitaliy Potyarkin --- pkg/local_object_storage/engine/evacuate_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index bd5222b78..359e49481 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -208,7 +208,7 @@ func testWaitForEvacuationCompleted(t *testing.T, e *StorageEngine) *EvacuationS st, err = e.GetEvacuationState(context.Background()) require.NoError(t, err) return st.ProcessingStatus() == EvacuateProcessStateCompleted - }, 3*time.Second, 10*time.Millisecond) + }, 6*time.Second, 10*time.Millisecond) return st } From fbc623f34e5d50d3ca16f1b63ba0ac5fbf011a0b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 2 Apr 2025 10:47:42 +0300 Subject: [PATCH 06/94] [#1701] go.mod: Bump `policy-engine` version Change-Id: I7aa359bf235034d6459275d366a276d9930fa227 Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 23be9c822..9ed5e4187 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/VictoriaMetrics/easyproto v0.1.4 diff --git a/go.sum b/go.sum index e5d476ffe..9727d8786 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/96 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972/go.mod h1:2hM42MBrlhvN6XToaW6OWNk5ZLcu1FhaukGgxtfpDDI= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07 h1:gPaqGsk6gSWQyNVjaStydfUz6Z/loHc9XyvGrJ5qSPY= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b h1:M50kdfrf/h8c3cz0bJ2AEUcbXvAlPFVC1Wp1WkfZ/8E= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b/go.mod h1:GZTk55RI4dKzsK6BCn5h2xxE28UHNfgoq/NJxW/LQ6A= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88 h1:V0a7ia84ZpSM2YxpJq1SKLQfeYmsqFWqcxwweBHJIzc= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88/go.mod h1:GZTk55RI4dKzsK6BCn5h2xxE28UHNfgoq/NJxW/LQ6A= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= From 979d4bb2aec9ce74a45b83f00defa532e5d764db Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 2 Apr 2025 11:15:40 +0300 Subject: [PATCH 07/94] [#1701] tree: Form `$Tree:ID` resource property for APE * Make `verifyClient`, `checkAPE` receive `treeID` from request body; * Make `newAPERequest` set `$Tree:ID` property * Add unit-test to check if a rule for `$Tree:ID` works Close #1701 Change-Id: I834fed366e8adfd4b5c07bf50aac09af6239991b Signed-off-by: Airat Arifullin --- pkg/services/tree/ape.go | 12 +++++--- pkg/services/tree/ape_test.go | 43 +++++++++++++++++++++++++++-- pkg/services/tree/service.go | 12 ++++---- pkg/services/tree/signature.go | 4 +-- pkg/services/tree/signature_test.go | 36 ++++++++++++------------ 5 files changed, 76 insertions(+), 31 deletions(-) diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index c4b03cbe6..58757ff6d 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -22,7 +22,7 @@ import ( ) func (s *Service) newAPERequest(ctx context.Context, namespace string, - cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, + cid cid.ID, treeID string, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, ) (aperequest.Request, error) { schemaMethod, err := converter.SchemaMethodFromACLOperation(operation) if err != nil { @@ -53,15 +53,19 @@ func (s *Service) newAPERequest(ctx context.Context, namespace string, resourceName = fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainerObjects, namespace, cid.EncodeToString()) } + resProps := map[string]string{ + nativeschema.ProperyKeyTreeID: treeID, + } + return aperequest.NewRequest( schemaMethod, - aperequest.NewResource(resourceName, make(map[string]string)), + aperequest.NewResource(resourceName, resProps), reqProps, ), nil } func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, - container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, + container *core.Container, cid cid.ID, treeID string, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, ) error { namespace := "" cntNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(container.Value).Zone(), ".ns") @@ -69,7 +73,7 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, namespace = cntNamespace } - request, err := s.newAPERequest(ctx, namespace, cid, operation, role, publicKey) + request, err := s.newAPERequest(ctx, namespace, cid, treeID, operation, role, publicKey) if err != nil { return fmt.Errorf("failed to create ape request: %w", err) } diff --git a/pkg/services/tree/ape_test.go b/pkg/services/tree/ape_test.go index 0afc7660a..7b209fd47 100644 --- a/pkg/services/tree/ape_test.go +++ b/pkg/services/tree/ape_test.go @@ -107,6 +107,45 @@ func TestCheckAPE(t *testing.T) { cid := cid.ID{} _ = cid.DecodeString(containerID) + t.Run("treeID rule", func(t *testing.T) { + los := inmemory.NewInmemoryLocalStorage() + mcs := inmemory.NewInmemoryMorphRuleChainStorage() + fid := newFrostfsIDProviderMock(t) + s := Service{ + cfg: cfg{ + frostfsidSubjectProvider: fid, + }, + apeChecker: checkercore.New(los, mcs, fid, &stMock{}), + } + + mcs.AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(containerID), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.QuotaLimitReached, + Actions: chain.Actions{Names: []string{nativeschema.MethodGetObject}}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + Condition: []chain.Condition{ + { + Op: chain.CondStringEquals, + Kind: chain.KindResource, + Key: nativeschema.ProperyKeyTreeID, + Value: versionTreeID, + }, + }, + }, + }, + MatchType: chain.MatchTypeFirstMatch, + }) + + err := s.checkAPE(context.Background(), nil, rootCnr, cid, versionTreeID, acl.OpObjectGet, acl.RoleOwner, senderPrivateKey.PublicKey()) + + var chErr *checkercore.ChainRouterError + require.ErrorAs(t, err, &chErr) + require.Equal(t, chain.QuotaLimitReached, chErr.Status()) + }) + t.Run("put non-tombstone rule won't affect tree remove", func(t *testing.T) { los := inmemory.NewInmemoryLocalStorage() mcs := inmemory.NewInmemoryMorphRuleChainStorage() @@ -152,7 +191,7 @@ func TestCheckAPE(t *testing.T) { MatchType: chain.MatchTypeFirstMatch, }) - err := s.checkAPE(context.Background(), nil, rootCnr, cid, acl.OpObjectDelete, acl.RoleOwner, senderPrivateKey.PublicKey()) + err := s.checkAPE(context.Background(), nil, rootCnr, cid, versionTreeID, acl.OpObjectDelete, acl.RoleOwner, senderPrivateKey.PublicKey()) require.NoError(t, err) }) @@ -201,7 +240,7 @@ func TestCheckAPE(t *testing.T) { MatchType: chain.MatchTypeFirstMatch, }) - err := s.checkAPE(context.Background(), nil, rootCnr, cid, acl.OpObjectPut, acl.RoleOwner, senderPrivateKey.PublicKey()) + err := s.checkAPE(context.Background(), nil, rootCnr, cid, versionTreeID, acl.OpObjectPut, acl.RoleOwner, senderPrivateKey.PublicKey()) require.NoError(t, err) }) } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index b9bb96bab..85bb03a28 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -117,7 +117,7 @@ func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectPut) if err != nil { return nil, err } @@ -161,7 +161,7 @@ func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByP return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectPut) if err != nil { return nil, err } @@ -217,7 +217,7 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectDelete) + err := s.verifyClient(ctx, req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectDelete) if err != nil { return nil, err } @@ -262,7 +262,7 @@ func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, er return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectPut) if err != nil { return nil, err } @@ -306,7 +306,7 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectGet) + err := s.verifyClient(ctx, req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectGet) if err != nil { return nil, err } @@ -377,7 +377,7 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS return err } - err := s.verifyClient(srv.Context(), req, cid, b.GetBearerToken(), acl.OpObjectGet) + err := s.verifyClient(srv.Context(), req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectGet) if err != nil { return err } diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index d15438e81..89056056d 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -38,7 +38,7 @@ var ( // Operation must be one of: // - 1. ObjectPut; // - 2. ObjectGet. -func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, rawBearer []byte, op acl.Op) error { +func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, treeID string, rawBearer []byte, op acl.Op) error { err := verifyMessage(req) if err != nil { return err @@ -64,7 +64,7 @@ func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, return fmt.Errorf("can't get request role: %w", err) } - if err = s.checkAPE(ctx, bt, cnr, cid, op, role, pubKey); err != nil { + if err = s.checkAPE(ctx, bt, cnr, cid, treeID, op, role, pubKey); err != nil { return apeErr(err) } return nil diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 97f8a727a..947de8e58 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -31,6 +31,8 @@ import ( "github.com/stretchr/testify/require" ) +const versionTreeID = "version" + type dummyNetmapSource struct { netmap.Source } @@ -168,26 +170,26 @@ func TestMessageSign(t *testing.T) { cnr.Value.SetBasicACL(acl.PublicRW) t.Run("missing signature, no panic", func(t *testing.T) { - require.Error(t, s.verifyClient(context.Background(), req, cid2, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid2, versionTreeID, nil, op)) }) require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, nil, op)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, nil, op)) t.Run("invalid CID", func(t *testing.T) { - require.Error(t, s.verifyClient(context.Background(), req, cid2, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid2, versionTreeID, nil, op)) }) cnr.Value.SetBasicACL(acl.Private) t.Run("extension disabled", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid2, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid2, versionTreeID, nil, op)) }) t.Run("invalid key", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, nil, op)) }) t.Run("bearer", func(t *testing.T) { @@ -200,7 +202,7 @@ func TestMessageSign(t *testing.T) { t.Run("invalid bearer", func(t *testing.T) { req.Body.BearerToken = []byte{0xFF} require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("invalid bearer CID", func(t *testing.T) { @@ -209,7 +211,7 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("invalid bearer owner", func(t *testing.T) { bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) @@ -217,7 +219,7 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("invalid bearer signature", func(t *testing.T) { bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) @@ -229,7 +231,7 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bv2.StableMarshal(nil) require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("impersonate", func(t *testing.T) { @@ -241,8 +243,8 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) @@ -252,18 +254,18 @@ func TestMessageSign(t *testing.T) { t.Run("put and get", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) t.Run("only get", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[2].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) t.Run("none", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[3].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) }) } From 52367dc9b259b6044f0fc7fda09e4339b7170f3d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 20:07:23 +0300 Subject: [PATCH 08/94] [#1689] go.mod: Update sdk-go Change-Id: I72052fe11e66e4c77f4aef6cb2c0f038aa7b0d1f Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9ed5e4187..5ed4a90be 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.5.2 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88 diff --git a/go.sum b/go.sum index 9727d8786..38aba9bde 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47 h1:O2c3VOlaGZ862hf2ZPLBMdTG6vGJzhIgDvFEFGfntzU= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 h1:zM2l316J55h9p30snl6vHBI/h0xmnuqZjnxIjRDtJZw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d h1:ZLKDupw362Ciing7kdIZhDYGMyo2QZyJ6sS/8X9QWJ0= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d/go.mod h1:2PWt5GwJTnhjHp+mankcfCeAJBMn7puxPm+RS+lliVk= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 3d771aa21c8fb961c53816566db77b3b0274dc40 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 5 Apr 2025 08:56:22 +0300 Subject: [PATCH 09/94] [#1689] morph: Remove `frostfsError` type It has no custom methods defined, only adds `frostfs error: ` prefix to the error message. The utility of this prefix is debatable, failed invocations already have `invocation failed` prefix. Change-Id: If25ebb3679497f3f10acde43b596c81d52351907 Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 18 ++---------------- pkg/morph/client/notary.go | 8 +++----- pkg/morph/client/util.go | 2 +- pkg/morph/client/waiter.go | 2 +- 4 files changed, 7 insertions(+), 23 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 1c12130b7..aab058d27 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -151,20 +151,6 @@ func (e *notHaltStateError) Error() string { ) } -// implementation of error interface for FrostFS-specific errors. -type frostfsError struct { - err error -} - -func (e frostfsError) Error() string { - return fmt.Sprintf("frostfs error: %v", e.err) -} - -// wraps FrostFS-specific error into frostfsError. Arg must not be nil. -func wrapFrostFSError(err error) error { - return frostfsError{err} -} - // Invoke invokes contract method by sending transaction into blockchain. // Returns valid until block value. // Supported args types: int64, string, util.Uint160, []byte and bool. @@ -228,7 +214,7 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int if err != nil { return err } else if val.State != HaltState { - return wrapFrostFSError(¬HaltStateError{state: val.State, exception: val.FaultException}) + return ¬HaltStateError{state: val.State, exception: val.FaultException} } arr, sid, r, err := unwrap.ArrayAndSessionIterator(val, err) @@ -292,7 +278,7 @@ func (c *Client) TestInvoke(contract util.Uint160, method string, args ...any) ( } if val.State != HaltState { - return nil, wrapFrostFSError(¬HaltStateError{state: val.State, exception: val.FaultException}) + return nil, ¬HaltStateError{state: val.State, exception: val.FaultException} } success = true diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 4e20a3639..448702613 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -461,7 +461,7 @@ func (c *Client) notaryInvoke(ctx context.Context, committee, invokedByAlpha boo mainH, fbH, untilActual, err := nAct.Notarize(nAct.MakeTunedCall(contract, method, nil, func(r *result.Invoke, t *transaction.Transaction) error { if r.State != vmstate.Halt.String() { - return wrapFrostFSError(¬HaltStateError{state: r.State, exception: r.FaultException}) + return ¬HaltStateError{state: r.State, exception: r.FaultException} } t.ValidUntilBlock = until @@ -608,8 +608,7 @@ func (c *Client) notaryMultisigAccount(ir []*keys.PublicKey, committee, invokedB multisigAccount = wallet.NewAccountFromPrivateKey(c.acc.PrivateKey()) err := multisigAccount.ConvertMultisig(m, ir) if err != nil { - // wrap error as FrostFS-specific since the call is not related to any client - return nil, wrapFrostFSError(fmt.Errorf("convert account to inner ring multisig wallet: %w", err)) + return nil, fmt.Errorf("convert account to inner ring multisig wallet: %w", err) } } else { // alphabet multisig redeem script is @@ -617,8 +616,7 @@ func (c *Client) notaryMultisigAccount(ir []*keys.PublicKey, committee, invokedB // inner ring multiaddress witness multisigAccount, err = notary.FakeMultisigAccount(m, ir) if err != nil { - // wrap error as FrostFS-specific since the call is not related to any client - return nil, wrapFrostFSError(fmt.Errorf("make inner ring multisig wallet: %w", err)) + return nil, fmt.Errorf("make inner ring multisig wallet: %w", err) } } diff --git a/pkg/morph/client/util.go b/pkg/morph/client/util.go index f68d39beb..f7b6705a8 100644 --- a/pkg/morph/client/util.go +++ b/pkg/morph/client/util.go @@ -98,7 +98,7 @@ func StringFromStackItem(param stackitem.Item) (string, error) { func addFeeCheckerModifier(add int64) func(r *result.Invoke, t *transaction.Transaction) error { return func(r *result.Invoke, t *transaction.Transaction) error { if r.State != HaltState { - return wrapFrostFSError(¬HaltStateError{state: r.State, exception: r.FaultException}) + return ¬HaltStateError{state: r.State, exception: r.FaultException} } t.SystemFee += add diff --git a/pkg/morph/client/waiter.go b/pkg/morph/client/waiter.go index 5b9d2cbe0..87fcf84b8 100644 --- a/pkg/morph/client/waiter.go +++ b/pkg/morph/client/waiter.go @@ -47,5 +47,5 @@ func (c *Client) WaitTxHalt(ctx context.Context, vub uint32, h util.Uint256) err if res.VMState.HasFlag(vmstate.Halt) { return nil } - return wrapFrostFSError(¬HaltStateError{state: res.VMState.String(), exception: res.FaultException}) + return ¬HaltStateError{state: res.VMState.String(), exception: res.FaultException} } From d933609084a23595e36bf48e5c226664d315bc23 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 5 Apr 2025 09:07:03 +0300 Subject: [PATCH 10/94] [#1689] client/netmap: Refactor Client.config() There are problems with that code: - explicit casts, - `ErrConfigNotFound` which is not a part of a public API, - hand-rolled assertions, even though neo-go already has everything we need. So, remove the error, use `stackitem/Item.Try*()` methods for conversions. Note, that readUint64Config() returns an error if the parameter is missing. This is likely an error, but this behaviour is preserved in this PR: `TryInteger()` returns error when applied to `Null`. By contract, `TryBool()` returns false for `Null`, so this PR introduces no functional changes. Refs https://github.com/nspcc-dev/neo-go/blob/82c7a50b8a308698e1440a716d34232094f9f55f/pkg/vm/stackitem/item.go#L418 Change-Id: I445d28a7c6b5abb9a2bb97b57c0cc42d617e16f7 Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/netmap/config.go | 46 +++++++------------------------ 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index fcdb70e3f..78063b857 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -2,7 +2,6 @@ package netmap import ( "context" - "errors" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -106,29 +105,27 @@ func (c *Client) MaintenanceModeAllowed(ctx context.Context) (bool, error) { } func (c *Client) readUInt64Config(ctx context.Context, key string) (uint64, error) { - v, err := c.config(ctx, []byte(key), IntegerAssert) + v, err := c.config(ctx, []byte(key)) if err != nil { return 0, fmt.Errorf("read netconfig value '%s': %w", key, err) } - // IntegerAssert is guaranteed to return int64 if the error is nil. - return uint64(v.(int64)), nil + bi, err := v.TryInteger() + if err != nil { + return 0, err + } + return bi.Uint64(), nil } // reads boolean value by the given key from the FrostFS network configuration // stored in the Sidechain. Returns false if key is not presented. func (c *Client) readBoolConfig(ctx context.Context, key string) (bool, error) { - v, err := c.config(ctx, []byte(key), BoolAssert) + v, err := c.config(ctx, []byte(key)) if err != nil { - if errors.Is(err, ErrConfigNotFound) { - return false, nil - } - return false, fmt.Errorf("read netconfig value '%s': %w", key, err) } - // BoolAssert is guaranteed to return bool if the error is nil. - return v.(bool), nil + return v.TryBool() } // SetConfigPrm groups parameters of SetConfig operation. @@ -277,15 +274,11 @@ func bytesToBool(val []byte) bool { return false } -// ErrConfigNotFound is returned when the requested key was not found -// in the network config (returned value is `Null`). -var ErrConfigNotFound = errors.New("config value not found") - // config performs the test invoke of get config value // method of FrostFS Netmap contract. // // Returns ErrConfigNotFound if config key is not found in the contract. -func (c *Client) config(ctx context.Context, key []byte, assert func(stackitem.Item) (any, error)) (any, error) { +func (c *Client) config(ctx context.Context, key []byte) (stackitem.Item, error) { prm := client.TestInvokePrm{} prm.SetMethod(configMethod) prm.SetArgs(key) @@ -301,26 +294,7 @@ func (c *Client) config(ctx context.Context, key []byte, assert func(stackitem.I configMethod, ln) } - if _, ok := items[0].(stackitem.Null); ok { - return nil, ErrConfigNotFound - } - - return assert(items[0]) -} - -// IntegerAssert converts stack item to int64. -func IntegerAssert(item stackitem.Item) (any, error) { - return client.IntFromStackItem(item) -} - -// StringAssert converts stack item to string. -func StringAssert(item stackitem.Item) (any, error) { - return client.StringFromStackItem(item) -} - -// BoolAssert converts stack item to bool. -func BoolAssert(item stackitem.Item) (any, error) { - return client.BoolFromStackItem(item) + return items[0], nil } // iterateRecords iterates over all config records and passes them to f. From c4f941a5f5217024ed8ba3cebda22d3518bd18f7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 5 Apr 2025 09:16:25 +0300 Subject: [PATCH 11/94] [#1689] client/netmap: Remove useless error-handling No functional changes. Change-Id: I3a53c992c3ce5e8c6db252abb09aa40626142a97 Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/netmap/config.go | 42 +++++-------------------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 78063b857..3f6aed506 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -25,44 +25,24 @@ const ( // MaxObjectSize receives max object size configuration // value through the Netmap contract call. func (c *Client) MaxObjectSize(ctx context.Context) (uint64, error) { - objectSize, err := c.readUInt64Config(ctx, MaxObjectSizeConfig) - if err != nil { - return 0, err - } - - return objectSize, nil + return c.readUInt64Config(ctx, MaxObjectSizeConfig) } // EpochDuration returns number of sidechain blocks per one FrostFS epoch. func (c *Client) EpochDuration(ctx context.Context) (uint64, error) { - epochDuration, err := c.readUInt64Config(ctx, EpochDurationConfig) - if err != nil { - return 0, err - } - - return epochDuration, nil + return c.readUInt64Config(ctx, EpochDurationConfig) } // ContainerFee returns fee paid by container owner to each alphabet node // for container registration. func (c *Client) ContainerFee(ctx context.Context) (uint64, error) { - fee, err := c.readUInt64Config(ctx, ContainerFeeConfig) - if err != nil { - return 0, err - } - - return fee, nil + return c.readUInt64Config(ctx, ContainerFeeConfig) } // ContainerAliasFee returns additional fee paid by container owner to each // alphabet node for container nice name registration. func (c *Client) ContainerAliasFee(ctx context.Context) (uint64, error) { - fee, err := c.readUInt64Config(ctx, ContainerAliasFeeConfig) - if err != nil { - return 0, err - } - - return fee, nil + return c.readUInt64Config(ctx, ContainerAliasFeeConfig) } // HomomorphicHashDisabled returns global configuration value of homomorphic hashing @@ -76,23 +56,13 @@ func (c *Client) HomomorphicHashDisabled(ctx context.Context) (bool, error) { // InnerRingCandidateFee returns global configuration value of fee paid by // node to be in inner ring candidates list. func (c *Client) InnerRingCandidateFee(ctx context.Context) (uint64, error) { - fee, err := c.readUInt64Config(ctx, IrCandidateFeeConfig) - if err != nil { - return 0, err - } - - return fee, nil + return c.readUInt64Config(ctx, IrCandidateFeeConfig) } // WithdrawFee returns global configuration value of fee paid by user to // withdraw assets from FrostFS contract. func (c *Client) WithdrawFee(ctx context.Context) (uint64, error) { - fee, err := c.readUInt64Config(ctx, WithdrawFeeConfig) - if err != nil { - return 0, err - } - - return fee, nil + return c.readUInt64Config(ctx, WithdrawFeeConfig) } // MaintenanceModeAllowed reads admission of "maintenance" state from the From 0c5d74729c62d535959965952fd88fe3fbdeffb2 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 13:43:30 +0300 Subject: [PATCH 12/94] [#1679] node: Fix 'revive' warning Change-Id: I74ff6332b10f17a329c5d108d01d43002e92aafd Signed-off-by: Alexander Chuprov --- pkg/util/ape/parser.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/util/ape/parser.go b/pkg/util/ape/parser.go index a34a17f6f..6f114d45b 100644 --- a/pkg/util/ape/parser.go +++ b/pkg/util/ape/parser.go @@ -174,11 +174,11 @@ func parseStatus(lexeme string) (apechain.Status, error) { case "deny": if !found { return apechain.AccessDenied, nil - } else if strings.EqualFold(expression, "QuotaLimitReached") { - return apechain.QuotaLimitReached, nil - } else { - return 0, fmt.Errorf("%w: %s", errUnknownStatusDetail, expression) } + if strings.EqualFold(expression, "QuotaLimitReached") { + return apechain.QuotaLimitReached, nil + } + return 0, fmt.Errorf("%w: %s", errUnknownStatusDetail, expression) case "allow": if found { return 0, errUnknownStatusDetail From b4b053cecd4c85361eee345b4b628e98412d270e Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 13:46:49 +0300 Subject: [PATCH 13/94] [#1679] node: Fix 'gocognit' warning Change-Id: I6e2a278af51869c05c306c2910ba85130e39532e Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/engine/lock.go | 29 ++++++++++++++++--------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 5d43e59df..18fff9cad 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -84,17 +84,11 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo var siErr *objectSDK.SplitInfoError var eiErr *objectSDK.ECInfoError if errors.As(err, &eiErr) { - eclocked := []oid.ID{locked} - for _, chunk := range eiErr.ECInfo().Chunks { - var objID oid.ID - err = objID.ReadFromV2(chunk.ID) - if err != nil { - e.reportShardError(ctx, sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), - zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) - return false - } - eclocked = append(eclocked, objID) + eclocked, ok := e.checkECLocked(ctx, sh, idCnr, locker, locked, eiErr) + if !ok { + return false } + err = sh.Lock(ctx, idCnr, locker, eclocked) if err != nil { e.reportShardError(ctx, sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), @@ -137,3 +131,18 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo }) return } + +func (e *StorageEngine) checkECLocked(ctx context.Context, sh hashedShard, idCnr cid.ID, locker, locked oid.ID, eiErr *objectSDK.ECInfoError) ([]oid.ID, bool) { + eclocked := []oid.ID{locked} + for _, chunk := range eiErr.ECInfo().Chunks { + var objID oid.ID + err := objID.ReadFromV2(chunk.ID) + if err != nil { + e.reportShardError(ctx, sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) + return nil, false + } + eclocked = append(eclocked, objID) + } + return eclocked, true +} From 9b5c1da40f2eac063ce38ca05d9211deba541234 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 14:21:44 +0300 Subject: [PATCH 14/94] [#1679] linter: Bump 'golangci-lint' to v1.64.8 - Removed deprecated config option 'linters.govet.check-shadowing', replaced with enabling the 'shadow' linter. - Removed usage of deprecated 'tenv' linter, replaced by 'usetesting'. Change-Id: Ib1bd1ec83b0fd55a47e405b290bc2bc967b9389c Signed-off-by: Alexander Chuprov --- .golangci.yml | 7 ++----- Makefile | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index f21a46248..18de49425 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -27,9 +27,6 @@ linters-settings: - standard - default custom-order: true - govet: - # report about shadowed variables - check-shadowing: false staticcheck: checks: ["all", "-SA1019"] # TODO Enable SA1019 after deprecated warning are fixed. funlen: @@ -60,8 +57,8 @@ linters-settings: linters: enable: # mandatory linters - - govet - revive + - predeclared # some default golangci-lint linters - errcheck @@ -94,8 +91,8 @@ linters: - testifylint - protogetter - intrange - - tenv - unconvert - unparam + - usetesting disable-all: true fast: false diff --git a/Makefile b/Makefile index 321365f0d..5b55c9eec 100755 --- a/Makefile +++ b/Makefile @@ -9,8 +9,8 @@ HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.23 -LINT_VERSION ?= 1.62.2 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.8 +LINT_VERSION ?= 1.64.8 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.10 PROTOC_VERSION ?= 25.0 PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-sdk-go) PROTOC_OS_VERSION=osx-x86_64 From 923f0acf8f03a485c665f9b610bd5f96b5c0200d Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 31 Mar 2025 05:12:22 +0300 Subject: [PATCH 15/94] [#1564] cli: Fix output of `object nodes` command The object nodes command misleadingly reported the number of "found data objects" as if it matched the actual expected amount, which could be incorrect for EC objects. Updated the output wording to explicitly distinguish between currently available data objects and total objects per the EC schema. Change-Id: Ib36b89db58ae66d8978baf5a16b59435db9a068d Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/modules/object/nodes.go | 59 ++++++++++++++++++------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index bc34b370d..734b557a4 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -48,6 +48,12 @@ type ecHeader struct { parent oid.ID } +type objectCounter struct { + sync.Mutex + total uint32 + isECcounted bool +} + type objectPlacement struct { requiredNodes []netmapSDK.NodeInfo confirmedNodes []netmapSDK.NodeInfo @@ -56,6 +62,7 @@ type objectPlacement struct { type objectNodesResult struct { errors []error placements map[oid.ID]objectPlacement + total uint32 } type ObjNodesDataObject struct { @@ -106,18 +113,18 @@ func objectNodes(cmd *cobra.Command, _ []string) { pk := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - objects := getPhyObjects(cmd, cnrID, objID, cli, pk) + objects, count := getPhyObjects(cmd, cnrID, objID, cli, pk) placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli) result := getRequiredPlacement(cmd, objects, placementPolicy, netmap) - getActualPlacement(cmd, netmap, pk, objects, result) + getActualPlacement(cmd, netmap, pk, objects, count, result) printPlacement(cmd, objID, objects, result) } -func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) []phyObject { +func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) ([]phyObject, int) { var addrObj oid.Address addrObj.SetContainer(cnrID) addrObj.SetObject(objID) @@ -145,7 +152,7 @@ func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.C parent: res.Header().ECHeader().Parent(), } } - return []phyObject{obj} + return []phyObject{obj}, 1 } var errSplitInfo *objectSDK.SplitInfoError @@ -155,29 +162,34 @@ func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.C var ecInfoError *objectSDK.ECInfoError if errors.As(err, &ecInfoError) { - return getECObjectChunks(cmd, cnrID, objID, ecInfoError) + return getECObjectChunks(cmd, cnrID, objID, ecInfoError), 1 } commonCmd.ExitOnErr(cmd, "failed to get object info: %w", err) - return nil + return nil, 0 } -func getComplexObjectParts(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) []phyObject { - members := getCompexObjectMembers(cmd, cnrID, objID, cli, prmHead, errSplitInfo) - return flattenComplexMembersIfECContainer(cmd, cnrID, members, prmHead) +func getComplexObjectParts(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) ([]phyObject, int) { + members, total := getCompexObjectMembers(cmd, cnrID, objID, cli, prmHead, errSplitInfo) + return flattenComplexMembersIfECContainer(cmd, cnrID, members, prmHead), total } -func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) []oid.ID { +func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) ([]oid.ID, int) { + var total int splitInfo := errSplitInfo.SplitInfo() if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID); ok { - return members + if total = len(members); total > 0 { + total-- // linking object is not data object + } + return members, total } if members, ok := tryGetSplitMembersBySplitID(cmd, splitInfo, cli, cnrID); ok { - return members + return members, len(members) } - return tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnrID, objID) + members := tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnrID, objID) + return members, len(members) } func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, members []oid.ID, prmHead internalclient.HeadObjectPrm) []phyObject { @@ -383,8 +395,11 @@ func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placem } } -func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa.PrivateKey, objects []phyObject, result *objectNodesResult) { +func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa.PrivateKey, objects []phyObject, count int, result *objectNodesResult) { resultMtx := &sync.Mutex{} + counter := &objectCounter{ + total: uint32(count), + } candidates := getNodesToCheckObjectExistance(cmd, netmap, result) @@ -401,7 +416,7 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa. for _, object := range objects { eg.Go(func() error { - stored, err := isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk) + stored, err := isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk, counter) resultMtx.Lock() defer resultMtx.Unlock() if err == nil && stored { @@ -420,6 +435,7 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa. } commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", eg.Wait()) + result.total = counter.total } func getNodesToCheckObjectExistance(cmd *cobra.Command, netmap *netmapSDK.NetMap, result *objectNodesResult) []netmapSDK.NodeInfo { @@ -478,7 +494,7 @@ func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.N return cli, nil } -func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) (bool, error) { +func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey, counter *objectCounter) (bool, error) { var addrObj oid.Address addrObj.SetContainer(cnrID) addrObj.SetObject(objID) @@ -493,6 +509,14 @@ func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, res, err := internalclient.HeadObject(ctx, prmHead) if err == nil && res != nil { + if res.Header().ECHeader() != nil { + counter.Lock() + defer counter.Unlock() + if !counter.isECcounted { + counter.total *= res.Header().ECHeader().Total() + } + counter.isECcounted = true + } return true, nil } var notFound *apistatus.ObjectNotFound @@ -512,7 +536,8 @@ func printPlacement(cmd *cobra.Command, objID oid.ID, objects []phyObject, resul } func printObjectNodesAsText(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) { - fmt.Fprintf(cmd.OutOrStdout(), "Object %s stores payload in %d data objects:\n", objID.EncodeToString(), len(objects)) + fmt.Fprintf(cmd.OutOrStdout(), "Object %s stores payload in %d data objects\n", objID.EncodeToString(), result.total) + fmt.Fprintf(cmd.OutOrStdout(), "Found %d:\n", len(objects)) for _, object := range objects { fmt.Fprintf(cmd.OutOrStdout(), "- %s\n", object.objectID) From 6f7b6b65f3b8cc3f372d900ee41242f0624b5b67 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 16:54:58 +0300 Subject: [PATCH 16/94] [#1689] linter: Fix staticcheck warning: 'embedded field can be simplified' Change-Id: I8f454f7d09973cdea096495c3949b88cdd01102e Signed-off-by: Alexander Chuprov --- .../modules/morph/contract/dump_hashes.go | 2 +- cmd/frostfs-lens/internal/tui/input.go | 8 ++-- cmd/frostfs-lens/internal/tui/ui.go | 2 +- cmd/frostfs-node/apemanager.go | 2 +- cmd/frostfs-node/container.go | 6 +-- cmd/frostfs-node/metrics.go | 28 ++++++------- cmd/frostfs-node/object.go | 6 +-- cmd/frostfs-node/pprof.go | 26 ++++++------ cmd/frostfs-node/tree.go | 4 +- pkg/local_object_storage/blobstor/blobstor.go | 2 +- pkg/local_object_storage/engine/container.go | 4 +- pkg/local_object_storage/engine/control.go | 2 +- pkg/local_object_storage/engine/inhume.go | 4 +- pkg/local_object_storage/engine/shards.go | 2 +- pkg/local_object_storage/pilorama/boltdb.go | 6 +-- pkg/local_object_storage/pilorama/forest.go | 4 +- pkg/local_object_storage/pilorama/inmemory.go | 10 ++--- pkg/local_object_storage/shard/control.go | 4 +- pkg/local_object_storage/shard/gc.go | 4 +- pkg/local_object_storage/shard/id.go | 2 +- pkg/local_object_storage/shard/shard.go | 42 +++++++++---------- pkg/services/control/server/evacuate_async.go | 2 +- pkg/services/object/common/writer/common.go | 6 +-- pkg/services/object/get/assemble.go | 2 +- pkg/services/object/get/get.go | 2 +- pkg/services/object/get/v2/streamer.go | 6 +-- pkg/services/object/patch/service.go | 2 +- pkg/services/object/put/single.go | 22 +++++----- pkg/services/policer/ec.go | 14 +++---- .../session/storage/persistent/storage.go | 2 +- pkg/services/tree/cache.go | 6 +-- pkg/services/tree/replicator.go | 2 +- pkg/services/tree/service.go | 2 +- pkg/services/tree/sync.go | 4 +- 34 files changed, 121 insertions(+), 121 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index fb7e4ff62..7630a226e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -242,7 +242,7 @@ func fillContractVersion(cmd *cobra.Command, c helper.Client, infos []contractDu script := sub.Bytes() emit.Instruction(bw.BinWriter, opcode.TRY, []byte{byte(3 + len(script) + 2), 0}) - bw.BinWriter.WriteBytes(script) + bw.WriteBytes(script) emit.Instruction(bw.BinWriter, opcode.ENDTRY, []byte{2 + 1}) emit.Opcodes(bw.BinWriter, opcode.PUSH0) } diff --git a/cmd/frostfs-lens/internal/tui/input.go b/cmd/frostfs-lens/internal/tui/input.go index 90729c119..471514e5d 100644 --- a/cmd/frostfs-lens/internal/tui/input.go +++ b/cmd/frostfs-lens/internal/tui/input.go @@ -53,17 +53,17 @@ func (f *InputFieldWithHistory) InputHandler() func(event *tcell.EventKey, setFo f.historyPointer++ // Stop iterating over history. if f.historyPointer == len(f.history) { - f.InputField.SetText(f.currentContent) + f.SetText(f.currentContent) return } - f.InputField.SetText(f.history[f.historyPointer]) + f.SetText(f.history[f.historyPointer]) case tcell.KeyUp: if len(f.history) == 0 { return } // Start iterating over history. if f.historyPointer == len(f.history) { - f.currentContent = f.InputField.GetText() + f.currentContent = f.GetText() } // End of history. if f.historyPointer == 0 { @@ -71,7 +71,7 @@ func (f *InputFieldWithHistory) InputHandler() func(event *tcell.EventKey, setFo } // Iterate to least recent prompts. f.historyPointer-- - f.InputField.SetText(f.history[f.historyPointer]) + f.SetText(f.history[f.historyPointer]) default: f.InputField.InputHandler()(event, func(tview.Primitive) {}) } diff --git a/cmd/frostfs-lens/internal/tui/ui.go b/cmd/frostfs-lens/internal/tui/ui.go index bcc082821..bd7540b01 100644 --- a/cmd/frostfs-lens/internal/tui/ui.go +++ b/cmd/frostfs-lens/internal/tui/ui.go @@ -482,7 +482,7 @@ func (ui *UI) handleInputOnSearching(event *tcell.EventKey) { ui.searchBar.InputHandler()(event, func(tview.Primitive) {}) } - ui.Box.MouseHandler() + ui.MouseHandler() } func (ui *UI) WithPrompt(prompt string) error { diff --git a/cmd/frostfs-node/apemanager.go b/cmd/frostfs-node/apemanager.go index e761a1b14..513314712 100644 --- a/cmd/frostfs-node/apemanager.go +++ b/cmd/frostfs-node/apemanager.go @@ -14,7 +14,7 @@ import ( func initAPEManagerService(c *cfg) { contractStorage := ape_contract.NewProxyVerificationContractStorage( morph.NewSwitchRPCGuardedActor(c.cfgMorph.client), - c.shared.key, + c.key, c.cfgMorph.proxyScriptHash, c.cfgObject.cfgAccessPolicyEngine.policyContractHash) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 012012297..bdb280d87 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -32,7 +32,7 @@ func initContainerService(_ context.Context, c *cfg) { wrap, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0) fatalOnErr(err) - c.shared.cnrClient = wrap + c.cnrClient = wrap cnrSrc := cntClient.AsContainerSource(wrap) @@ -47,7 +47,7 @@ func initContainerService(_ context.Context, c *cfg) { frostfsIDSubjectProvider = newMorphFrostfsIDCache(frostfsIDSubjectProvider, int(cacheSize), c.cfgMorph.cacheTTL, metrics.NewCacheMetrics("frostfs_id")) } - c.shared.frostfsidClient = frostfsIDSubjectProvider + c.frostfsidClient = frostfsIDSubjectProvider c.cfgContainer.containerBatchSize = containerconfig.ContainerBatchSize(c.appCfg) defaultChainRouter := engine.NewDefaultChainRouterWithLocalOverrides( @@ -57,7 +57,7 @@ func initContainerService(_ context.Context, c *cfg) { service := containerService.NewSignService( &c.key.PrivateKey, containerService.NewAPEServer(defaultChainRouter, cnrRdr, - newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, c.shared.frostfsidClient, + newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, c.frostfsidClient, containerService.NewSplitterService( c.cfgContainer.containerBatchSize, c.respSvc, containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc)), diff --git a/cmd/frostfs-node/metrics.go b/cmd/frostfs-node/metrics.go index 19b4af51f..d9ca01e70 100644 --- a/cmd/frostfs-node/metrics.go +++ b/cmd/frostfs-node/metrics.go @@ -8,38 +8,38 @@ import ( func metricsComponent(c *cfg) (*httpComponent, bool) { var updated bool // check if it has been inited before - if c.dynamicConfiguration.metrics == nil { - c.dynamicConfiguration.metrics = new(httpComponent) - c.dynamicConfiguration.metrics.cfg = c - c.dynamicConfiguration.metrics.name = "metrics" - c.dynamicConfiguration.metrics.handler = metrics.Handler() + if c.metrics == nil { + c.metrics = new(httpComponent) + c.metrics.cfg = c + c.metrics.name = "metrics" + c.metrics.handler = metrics.Handler() updated = true } // (re)init read configuration enabled := metricsconfig.Enabled(c.appCfg) - if enabled != c.dynamicConfiguration.metrics.enabled { - c.dynamicConfiguration.metrics.enabled = enabled + if enabled != c.metrics.enabled { + c.metrics.enabled = enabled updated = true } address := metricsconfig.Address(c.appCfg) - if address != c.dynamicConfiguration.metrics.address { - c.dynamicConfiguration.metrics.address = address + if address != c.metrics.address { + c.metrics.address = address updated = true } dur := metricsconfig.ShutdownTimeout(c.appCfg) - if dur != c.dynamicConfiguration.metrics.shutdownDur { - c.dynamicConfiguration.metrics.shutdownDur = dur + if dur != c.metrics.shutdownDur { + c.metrics.shutdownDur = dur updated = true } - return c.dynamicConfiguration.metrics, updated + return c.metrics, updated } func enableMetricsSvc(c *cfg) { - c.shared.metricsSvc.Enable() + c.metricsSvc.Enable() } func disableMetricsSvc(c *cfg) { - c.shared.metricsSvc.Disable() + c.metricsSvc.Disable() } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 2674be8c7..527746d26 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -186,9 +186,9 @@ func initObjectService(c *cfg) { respSvc, ) - c.shared.metricsSvc = objectService.NewMetricCollector( + c.metricsSvc = objectService.NewMetricCollector( signSvc, c.metricsCollector.ObjectService(), metricsconfig.Enabled(c.appCfg)) - qosService := objectService.NewQoSObjectService(c.shared.metricsSvc, &c.cfgQoSService) + qosService := objectService.NewQoSObjectService(c.metricsSvc, &c.cfgQoSService) auditSvc := objectService.NewAuditService(qosService, c.log, c.audit) server := objectTransportGRPC.New(auditSvc) @@ -432,7 +432,7 @@ func createAPEService(c *cfg, irFetcher *cachedIRFetcher, splitSvc *objectServic c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalStorage(), c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.MorphRuleChainStorage(), objectAPE.NewStorageEngineHeaderProvider(c.cfgObject.cfgLocalStorage.localStorage, c.cfgObject.getSvc), - c.shared.frostfsidClient, + c.frostfsidClient, c.netMapSource, c.cfgNetmap.state, c.cfgObject.cnrSource, diff --git a/cmd/frostfs-node/pprof.go b/cmd/frostfs-node/pprof.go index 5b40c8a88..e4da8119f 100644 --- a/cmd/frostfs-node/pprof.go +++ b/cmd/frostfs-node/pprof.go @@ -18,33 +18,33 @@ func initProfilerService(ctx context.Context, c *cfg) { func pprofComponent(c *cfg) (*httpComponent, bool) { var updated bool // check if it has been inited before - if c.dynamicConfiguration.pprof == nil { - c.dynamicConfiguration.pprof = new(httpComponent) - c.dynamicConfiguration.pprof.cfg = c - c.dynamicConfiguration.pprof.name = "pprof" - c.dynamicConfiguration.pprof.handler = httputil.Handler() - c.dynamicConfiguration.pprof.preReload = tuneProfilers + if c.pprof == nil { + c.pprof = new(httpComponent) + c.pprof.cfg = c + c.pprof.name = "pprof" + c.pprof.handler = httputil.Handler() + c.pprof.preReload = tuneProfilers updated = true } // (re)init read configuration enabled := profilerconfig.Enabled(c.appCfg) - if enabled != c.dynamicConfiguration.pprof.enabled { - c.dynamicConfiguration.pprof.enabled = enabled + if enabled != c.pprof.enabled { + c.pprof.enabled = enabled updated = true } address := profilerconfig.Address(c.appCfg) - if address != c.dynamicConfiguration.pprof.address { - c.dynamicConfiguration.pprof.address = address + if address != c.pprof.address { + c.pprof.address = address updated = true } dur := profilerconfig.ShutdownTimeout(c.appCfg) - if dur != c.dynamicConfiguration.pprof.shutdownDur { - c.dynamicConfiguration.pprof.shutdownDur = dur + if dur != c.pprof.shutdownDur { + c.pprof.shutdownDur = dur updated = true } - return c.dynamicConfiguration.pprof, updated + return c.pprof, updated } func tuneProfilers(c *cfg) { diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 65414f0ca..67d9c9df0 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -51,9 +51,9 @@ func initTreeService(c *cfg) { c.treeService = tree.New( tree.WithContainerSource(cnrSource{ src: c.cfgObject.cnrSource, - cli: c.shared.cnrClient, + cli: c.cnrClient, }), - tree.WithFrostfsidSubjectProvider(c.shared.frostfsidClient), + tree.WithFrostfsidSubjectProvider(c.frostfsidClient), tree.WithNetmapSource(c.netMapSource), tree.WithPrivateKey(&c.key.PrivateKey), tree.WithLogger(c.log), diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index f850f48b4..d7c333349 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -153,5 +153,5 @@ func WithMetrics(m Metrics) Option { } func (b *BlobStor) Compressor() *compression.Config { - return &b.cfg.compression + return &b.compression } diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 3160d7f83..03a299b93 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -74,7 +74,7 @@ func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) var csPrm shard.ContainerSizePrm csPrm.SetContainerID(prm.cnr) - csRes, err := sh.Shard.ContainerSize(ctx, csPrm) + csRes, err := sh.ContainerSize(ctx, csPrm) if err != nil { e.reportShardError(ctx, sh, "can't get container size", err, zap.Stringer("container_id", prm.cnr)) @@ -119,7 +119,7 @@ func (e *StorageEngine) listContainers(ctx context.Context) ListContainersRes { uniqueIDs := make(map[string]cid.ID) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - res, err := sh.Shard.ListContainers(ctx, shard.ListContainersPrm{}) + res, err := sh.ListContainers(ctx, shard.ListContainersPrm{}) if err != nil { e.reportShardError(ctx, sh, "can't get list of containers", err) return false diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 7caa515d4..96b53581e 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -77,7 +77,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { errCh := make(chan shardInitError, len(e.shards)) var eg errgroup.Group - if e.cfg.lowMem && e.anyShardRequiresRefill() { + if e.lowMem && e.anyShardRequiresRefill() { eg.SetLimit(1) } diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index c8ee33b53..6ec9a4ef0 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -227,7 +227,7 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e var outErr error e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { - locked, err = h.Shard.IsLocked(ctx, addr) + locked, err = h.IsLocked(ctx, addr) if err != nil { e.reportShardError(ctx, h, "can't check object's lockers", err, zap.Stringer("address", addr)) outErr = err @@ -256,7 +256,7 @@ func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.I var outErr error e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { - locks, err := h.Shard.GetLocks(ctx, addr) + locks, err := h.GetLocks(ctx, addr) if err != nil { e.reportShardError(ctx, h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr)) outErr = err diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index dfc3b1a35..6e6c08bb5 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -118,7 +118,7 @@ func (e *StorageEngine) AddShard(ctx context.Context, opts ...shard.Option) (*sh return nil, fmt.Errorf("add %s shard: %w", sh.ID().String(), err) } - e.cfg.metrics.SetMode(sh.ID().String(), sh.GetMode()) + e.metrics.SetMode(sh.ID().String(), sh.GetMode()) return sh.ID(), nil } diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index fc7cdaabc..897b37ea0 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1582,12 +1582,12 @@ func (t *boltForest) moveFromBytes(m *Move, data []byte) error { func (t *boltForest) logFromBytes(lm *Move, data []byte) error { lm.Child = binary.LittleEndian.Uint64(data) lm.Parent = binary.LittleEndian.Uint64(data[8:]) - return lm.Meta.FromBytes(data[16:]) + return lm.FromBytes(data[16:]) } func (t *boltForest) logToBytes(lm *Move) []byte { w := io.NewBufBinWriter() - size := 8 + 8 + lm.Meta.Size() + 1 + size := 8 + 8 + lm.Size() + 1 // if lm.HasOld { // size += 8 + lm.Old.Meta.Size() // } @@ -1595,7 +1595,7 @@ func (t *boltForest) logToBytes(lm *Move) []byte { w.Grow(size) w.WriteU64LE(lm.Child) w.WriteU64LE(lm.Parent) - lm.Meta.EncodeBinary(w.BinWriter) + lm.EncodeBinary(w.BinWriter) // w.WriteBool(lm.HasOld) // if lm.HasOld { // w.WriteU64LE(lm.Old.Parent) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index b5320e42d..ebfd0bcc0 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -177,7 +177,7 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI var res []NodeInfo for _, nodeID := range nodeIDs { - children := s.tree.getChildren(nodeID) + children := s.getChildren(nodeID) for _, childID := range children { var found bool for _, kv := range s.infoMap[childID].Meta.Items { @@ -222,7 +222,7 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str return nil, ErrTreeNotFound } - children := s.tree.getChildren(nodeID) + children := s.getChildren(nodeID) res := make([]NodeInfo, 0, len(children)) for _, childID := range children { res = append(res, NodeInfo{ diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index ce7b3db1e..f74d12a1b 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -35,9 +35,9 @@ func newMemoryTree() *memoryTree { // undo un-does op and changes s in-place. func (s *memoryTree) undo(op *move) { if op.HasOld { - s.tree.infoMap[op.Child] = op.Old + s.infoMap[op.Child] = op.Old } else { - delete(s.tree.infoMap, op.Child) + delete(s.infoMap, op.Child) } } @@ -83,8 +83,8 @@ func (s *memoryTree) do(op *Move) move { }, } - shouldPut := !s.tree.isAncestor(op.Child, op.Parent) - p, ok := s.tree.infoMap[op.Child] + shouldPut := !s.isAncestor(op.Child, op.Parent) + p, ok := s.infoMap[op.Child] if ok { lm.HasOld = true lm.Old = p @@ -100,7 +100,7 @@ func (s *memoryTree) do(op *Move) move { p.Meta = m p.Parent = op.Parent - s.tree.infoMap[op.Child] = p + s.infoMap[op.Child] = p return lm } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 19b13a8ab..72e650c5e 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -214,8 +214,8 @@ func (s *Shard) refillMetabase(ctx context.Context) error { } eg, egCtx := errgroup.WithContext(ctx) - if s.cfg.refillMetabaseWorkersCount > 0 { - eg.SetLimit(s.cfg.refillMetabaseWorkersCount) + if s.refillMetabaseWorkersCount > 0 { + eg.SetLimit(s.refillMetabaseWorkersCount) } var completedCount uint64 diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 9b327f6f1..19b6e2d12 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -320,8 +320,8 @@ func (s *Shard) getGarbage(ctx context.Context) ([]oid.Address, error) { } func (s *Shard) getExpiredObjectsParameters() (workerCount, batchSize int) { - workerCount = max(minExpiredWorkers, s.gc.gcCfg.expiredCollectorWorkerCount) - batchSize = max(minExpiredBatchSize, s.gc.gcCfg.expiredCollectorBatchSize) + workerCount = max(minExpiredWorkers, s.gc.expiredCollectorWorkerCount) + batchSize = max(minExpiredBatchSize, s.gc.expiredCollectorBatchSize) return } diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index b233b705c..7391adef2 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -45,7 +45,7 @@ func (s *Shard) UpdateID(ctx context.Context) (err error) { } shardID := s.info.ID.String() - s.cfg.metricsWriter.SetShardID(shardID) + s.metricsWriter.SetShardID(shardID) if s.writeCache != nil && s.writeCache.GetMetrics() != nil { s.writeCache.GetMetrics().SetShardID(shardID) } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index b9ec05f01..304a6bf9d 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -218,7 +218,7 @@ func WithWriteCache(use bool) Option { // hasWriteCache returns bool if write cache exists on shards. func (s *Shard) hasWriteCache() bool { - return s.cfg.useWriteCache + return s.useWriteCache } // NeedRefillMetabase returns true if metabase is needed to be refilled. @@ -379,15 +379,15 @@ func WithLimiter(l qos.Limiter) Option { } func (s *Shard) fillInfo() { - s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo() - s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo() - s.cfg.info.Mode = s.GetMode() + s.info.MetaBaseInfo = s.metaBase.DumpInfo() + s.info.BlobStorInfo = s.blobStor.DumpInfo() + s.info.Mode = s.GetMode() - if s.cfg.useWriteCache { - s.cfg.info.WriteCacheInfo = s.writeCache.DumpInfo() + if s.useWriteCache { + s.info.WriteCacheInfo = s.writeCache.DumpInfo() } if s.pilorama != nil { - s.cfg.info.PiloramaInfo = s.pilorama.DumpInfo() + s.info.PiloramaInfo = s.pilorama.DumpInfo() } } @@ -454,57 +454,57 @@ func (s *Shard) updateMetrics(ctx context.Context) { s.setContainerObjectsCount(contID.EncodeToString(), logical, count.Logic) s.setContainerObjectsCount(contID.EncodeToString(), user, count.User) } - s.cfg.metricsWriter.SetMode(s.info.Mode) + s.metricsWriter.SetMode(s.info.Mode) } // incObjectCounter increment both physical and logical object // counters. func (s *Shard) incObjectCounter(cnrID cid.ID, isUser bool) { - s.cfg.metricsWriter.IncObjectCounter(physical) - s.cfg.metricsWriter.IncObjectCounter(logical) - s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), physical) - s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), logical) + s.metricsWriter.IncObjectCounter(physical) + s.metricsWriter.IncObjectCounter(logical) + s.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), physical) + s.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), logical) if isUser { - s.cfg.metricsWriter.IncObjectCounter(user) - s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), user) + s.metricsWriter.IncObjectCounter(user) + s.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), user) } } func (s *Shard) decObjectCounterBy(typ string, v uint64) { if v > 0 { - s.cfg.metricsWriter.AddToObjectCounter(typ, -int(v)) + s.metricsWriter.AddToObjectCounter(typ, -int(v)) } } func (s *Shard) setObjectCounterBy(typ string, v uint64) { if v > 0 { - s.cfg.metricsWriter.SetObjectCounter(typ, v) + s.metricsWriter.SetObjectCounter(typ, v) } } func (s *Shard) decContainerObjectCounter(byCnr map[cid.ID]meta.ObjectCounters) { for cnrID, count := range byCnr { if count.Phy > 0 { - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), physical, count.Phy) + s.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), physical, count.Phy) } if count.Logic > 0 { - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), logical, count.Logic) + s.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), logical, count.Logic) } if count.User > 0 { - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), user, count.User) + s.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), user, count.User) } } } func (s *Shard) addToContainerSize(cnr string, size int64) { if size != 0 { - s.cfg.metricsWriter.AddToContainerSize(cnr, size) + s.metricsWriter.AddToContainerSize(cnr, size) } } func (s *Shard) addToPayloadSize(size int64) { if size != 0 { - s.cfg.metricsWriter.AddToPayloadSize(size) + s.metricsWriter.AddToPayloadSize(size) } } diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 7469ea74e..f3ba9015e 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -220,7 +220,7 @@ func (s *Server) replicateTreeToNode(ctx context.Context, forest pilorama.Forest TreeId: treeID, Operation: &tree.LogMove{ ParentId: op.Parent, - Meta: op.Meta.Bytes(), + Meta: op.Bytes(), ChildId: op.Child, }, }, diff --git a/pkg/services/object/common/writer/common.go b/pkg/services/object/common/writer/common.go index 1998e9638..6593d3ca0 100644 --- a/pkg/services/object/common/writer/common.go +++ b/pkg/services/object/common/writer/common.go @@ -29,7 +29,7 @@ func (c *Config) NewNodeIterator(opts []placement.Option) *NodeIterator { } func (n *NodeIterator) ForEachNode(ctx context.Context, f func(context.Context, NodeDescriptor) error) error { - traverser, err := placement.NewTraverser(ctx, n.Traversal.Opts...) + traverser, err := placement.NewTraverser(ctx, n.Opts...) if err != nil { return fmt.Errorf("could not create object placement traverser: %w", err) } @@ -56,7 +56,7 @@ func (n *NodeIterator) ForEachNode(ctx context.Context, f func(context.Context, } // perform additional container broadcast if needed - if n.Traversal.submitPrimaryPlacementFinish() { + if n.submitPrimaryPlacementFinish() { err := n.ForEachNode(ctx, f) if err != nil { n.cfg.Logger.Error(ctx, logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) @@ -101,7 +101,7 @@ func (n *NodeIterator) forEachAddress(ctx context.Context, traverser *placement. // in subsequent container broadcast. Note that we don't // process this node during broadcast if primary placement // on it failed. - n.Traversal.submitProcessed(addr, item) + n.submitProcessed(addr, item) } wg.Wait() diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index e164627d2..e80132489 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -146,5 +146,5 @@ func (r *request) getObjectWithIndependentRequest(ctx context.Context, prm Reque detachedExecutor.execute(ctx) - return detachedExecutor.statusError.err + return detachedExecutor.err } diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 557e9a028..9676fd914 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -87,7 +87,7 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { exec.execute(ctx) - return exec.statusError.err + return exec.err } func (exec *request) execute(ctx context.Context) { diff --git a/pkg/services/object/get/v2/streamer.go b/pkg/services/object/get/v2/streamer.go index 98207336c..0d73bcd4d 100644 --- a/pkg/services/object/get/v2/streamer.go +++ b/pkg/services/object/get/v2/streamer.go @@ -24,14 +24,14 @@ func (s *streamObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Objec p.SetHeader(objV2.GetHeader()) p.SetSignature(objV2.GetSignature()) - return s.GetObjectStream.Send(newResponse(p)) + return s.Send(newResponse(p)) } func (s *streamObjectWriter) WriteChunk(_ context.Context, chunk []byte) error { p := new(objectV2.GetObjectPartChunk) p.SetChunk(chunk) - return s.GetObjectStream.Send(newResponse(p)) + return s.Send(newResponse(p)) } func newResponse(p objectV2.GetObjectPart) *objectV2.GetResponse { @@ -46,7 +46,7 @@ func newResponse(p objectV2.GetObjectPart) *objectV2.GetResponse { } func (s *streamObjectRangeWriter) WriteChunk(_ context.Context, chunk []byte) error { - return s.GetObjectRangeStream.Send(newRangeResponse(chunk)) + return s.Send(newRangeResponse(chunk)) } func newRangeResponse(p []byte) *objectV2.GetRangeResponse { diff --git a/pkg/services/object/patch/service.go b/pkg/services/object/patch/service.go index 953f82b48..5d298bfed 100644 --- a/pkg/services/object/patch/service.go +++ b/pkg/services/object/patch/service.go @@ -28,7 +28,7 @@ func NewService(cfg *objectwriter.Config, // Patch calls internal service and returns v2 object streamer. func (s *Service) Patch() (object.PatchObjectStream, error) { - nodeKey, err := s.Config.KeyStorage.GetKey(nil) + nodeKey, err := s.KeyStorage.GetKey(nil) if err != nil { return nil, err } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index f3c2dca1a..90f473254 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -102,7 +102,7 @@ func (s *Service) validarePutSingleSize(ctx context.Context, obj *objectSDK.Obje return target.ErrWrongPayloadSize } - maxAllowedSize := s.Config.MaxSizeSrc.MaxObjectSize(ctx) + maxAllowedSize := s.MaxSizeSrc.MaxObjectSize(ctx) if obj.PayloadSize() > maxAllowedSize { return target.ErrExceedingMaxSize } @@ -166,13 +166,13 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o } func (s *Service) saveToREPReplicas(ctx context.Context, placement putSinglePlacement, obj *objectSDK.Object, localOnly bool, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { - iter := s.Config.NewNodeIterator(placement.placementOptions) + iter := s.NewNodeIterator(placement.placementOptions) iter.ExtraBroadcastEnabled = objectwriter.NeedAdditionalBroadcast(obj, localOnly) iter.ResetSuccessAfterOnBroadcast = placement.resetSuccessAfterOnBroadcast signer := &putSingleRequestSigner{ req: req, - keyStorage: s.Config.KeyStorage, + keyStorage: s.KeyStorage, signer: &sync.Once{}, } @@ -186,13 +186,13 @@ func (s *Service) saveToECReplicas(ctx context.Context, placement putSinglePlace if err != nil { return err } - key, err := s.Config.KeyStorage.GetKey(nil) + key, err := s.KeyStorage.GetKey(nil) if err != nil { return err } signer := &putSingleRequestSigner{ req: req, - keyStorage: s.Config.KeyStorage, + keyStorage: s.KeyStorage, signer: &sync.Once{}, } @@ -225,7 +225,7 @@ func (s *Service) getPutSinglePlacementOptions(ctx context.Context, obj *objectS if !ok { return result, errors.New("missing container ID") } - cnrInfo, err := s.Config.ContainerSource.Get(ctx, cnrID) + cnrInfo, err := s.ContainerSource.Get(ctx, cnrID) if err != nil { return result, fmt.Errorf("could not get container by ID: %w", err) } @@ -249,14 +249,14 @@ func (s *Service) getPutSinglePlacementOptions(ctx context.Context, obj *objectS } result.placementOptions = append(result.placementOptions, placement.ForObject(objID)) - latestNetmap, err := netmap.GetLatestNetworkMap(ctx, s.Config.NetmapSource) + latestNetmap, err := netmap.GetLatestNetworkMap(ctx, s.NetmapSource) if err != nil { return result, fmt.Errorf("could not get latest network map: %w", err) } builder := placement.NewNetworkMapBuilder(latestNetmap) if localOnly { result.placementOptions = append(result.placementOptions, placement.SuccessAfter(1)) - builder = svcutil.NewLocalPlacement(builder, s.Config.NetmapKeys) + builder = svcutil.NewLocalPlacement(builder, s.NetmapKeys) } result.placementOptions = append(result.placementOptions, placement.UseBuilder(builder)) return result, nil @@ -273,7 +273,7 @@ func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *objectwrite client.NodeInfoFromNetmapElement(&info, nodeDesc.Info) - c, err := s.Config.ClientConstructor.Get(info) + c, err := s.ClientConstructor.Get(info) if err != nil { return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) } @@ -283,7 +283,7 @@ func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *objectwrite func (s *Service) saveLocal(ctx context.Context, obj *objectSDK.Object, meta object.ContentMeta, container containerSDK.Container) error { localTarget := &objectwriter.LocalTarget{ - Storage: s.Config.LocalStore, + Storage: s.LocalStore, Container: container, } return localTarget.WriteObject(ctx, obj, meta) @@ -317,7 +317,7 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, if err != nil { objID, _ := obj.ID() cnrID, _ := obj.ContainerID() - s.Config.Logger.Warn(ctx, logs.PutSingleRedirectFailure, + s.Logger.Warn(ctx, logs.PutSingleRedirectFailure, zap.Error(err), zap.Stringer("address", addr), zap.Stringer("object_id", objID), diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index 1ee31d480..fbdeb3148 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -101,7 +101,7 @@ func (p *Policer) processECContainerECObject(ctx context.Context, objInfo object func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) ecChunkProcessResult { var removeLocalChunk bool requiredNode := nodes[int(objInfo.ECInfo.Index)%(len(nodes))] - if p.cfg.netmapKeys.IsLocalKey(requiredNode.PublicKey()) { + if p.netmapKeys.IsLocalKey(requiredNode.PublicKey()) { // current node is required node, we are happy return ecChunkProcessResult{ validPlacement: true, @@ -185,7 +185,7 @@ func (p *Policer) collectRequiredECChunks(nodes []netmap.NodeInfo, objInfo objec if uint32(i) == objInfo.ECInfo.Total { break } - if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { + if p.netmapKeys.IsLocalKey(n.PublicKey()) { requiredChunkIndexes[uint32(i)] = []netmap.NodeInfo{} } } @@ -210,7 +210,7 @@ func (p *Policer) resolveLocalECChunks(ctx context.Context, parentAddress oid.Ad func (p *Policer) resolveRemoteECChunks(ctx context.Context, parentAddress oid.Address, nodes []netmap.NodeInfo, required map[uint32][]netmap.NodeInfo, indexToObjectID map[uint32]oid.ID) bool { var eiErr *objectSDK.ECInfoError for _, n := range nodes { - if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { + if p.netmapKeys.IsLocalKey(n.PublicKey()) { continue } _, err := p.remoteHeader(ctx, n, parentAddress, true) @@ -260,7 +260,7 @@ func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info return } var err error - if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { + if p.netmapKeys.IsLocalKey(n.PublicKey()) { _, err = p.localHeader(ctx, parentAddress) } else { _, err = p.remoteHeader(ctx, n, parentAddress, true) @@ -283,7 +283,7 @@ func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info } } else if client.IsErrObjectAlreadyRemoved(err) { restore = false - } else if !p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) && uint32(idx) < objInfo.ECInfo.Total { + } else if !p.netmapKeys.IsLocalKey(n.PublicKey()) && uint32(idx) < objInfo.ECInfo.Total { p.log.Warn(ctx, logs.PolicerCouldNotGetObjectFromNodeMoving, zap.String("node", hex.EncodeToString(n.PublicKey())), zap.Stringer("object", parentAddress), zap.Error(err)) p.replicator.HandleReplicationTask(ctx, replicator.Task{ NumCopies: 1, @@ -343,7 +343,7 @@ func (p *Policer) restoreECObject(ctx context.Context, objInfo objectcore.Info, pID, _ := part.ID() addr.SetObject(pID) targetNode := nodes[idx%len(nodes)] - if p.cfg.netmapKeys.IsLocalKey(targetNode.PublicKey()) { + if p.netmapKeys.IsLocalKey(targetNode.PublicKey()) { p.replicator.HandleLocalPutTask(ctx, replicator.Task{ Addr: addr, Obj: part, @@ -371,7 +371,7 @@ func (p *Policer) collectExistedChunks(ctx context.Context, objInfo objectcore.I var obj *objectSDK.Object var err error for _, node := range nodes { - if p.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { + if p.netmapKeys.IsLocalKey(node.PublicKey()) { obj, err = p.localObject(egCtx, objID) } else { obj, err = p.remoteObject(egCtx, node, objID) diff --git a/pkg/services/session/storage/persistent/storage.go b/pkg/services/session/storage/persistent/storage.go index d312ea0ea..132d62445 100644 --- a/pkg/services/session/storage/persistent/storage.go +++ b/pkg/services/session/storage/persistent/storage.go @@ -64,7 +64,7 @@ func NewTokenStore(path string, opts ...Option) (*TokenStore, error) { // enable encryption if it // was configured so if cfg.privateKey != nil { - rawKey := make([]byte, (cfg.privateKey.Curve.Params().N.BitLen()+7)/8) + rawKey := make([]byte, (cfg.privateKey.Params().N.BitLen()+7)/8) cfg.privateKey.D.FillBytes(rawKey) c, err := aes.NewCipher(rawKey) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index d250f577a..462c8554f 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -48,7 +48,7 @@ func (c *clientCache) init(pk *ecdsa.PrivateKey, ds *internalNet.DialerSource) { func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceClient, error) { c.Lock() - ccInt, ok := c.LRU.Get(netmapAddr) + ccInt, ok := c.Get(netmapAddr) c.Unlock() if ok { @@ -71,9 +71,9 @@ func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceCl c.Lock() if err != nil { - c.LRU.Add(netmapAddr, cacheItem{cc: nil, lastTry: lastTry}) + c.Add(netmapAddr, cacheItem{cc: nil, lastTry: lastTry}) } else { - c.LRU.Add(netmapAddr, cacheItem{cc: cc, lastTry: lastTry}) + c.Add(netmapAddr, cacheItem{cc: cc, lastTry: lastTry}) } c.Unlock() diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 164815c76..01a4ffde0 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -202,7 +202,7 @@ func newApplyRequest(op *movePair) *ApplyRequest { TreeId: op.treeID, Operation: &LogMove{ ParentId: op.op.Parent, - Meta: op.op.Meta.Bytes(), + Meta: op.op.Bytes(), ChildId: op.op.Child, }, }, diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 85bb03a28..a00c8c1cd 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -687,7 +687,7 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) Body: &GetOpLogResponse_Body{ Operation: &LogMove{ ParentId: lm.Parent, - Meta: lm.Meta.Bytes(), + Meta: lm.Bytes(), ChildId: lm.Child, }, }, diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 8abdafaa7..c3796fbd4 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -245,7 +245,7 @@ func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, Parent: lm.GetParentId(), Child: lm.GetChildId(), } - if err := m.Meta.FromBytes(lm.GetMeta()); err != nil { + if err := m.FromBytes(lm.GetMeta()); err != nil { return err } select { @@ -415,7 +415,7 @@ func (s *Service) syncLoop(ctx context.Context) { start := time.Now() - cnrs, err := s.cfg.cnrSource.List(ctx) + cnrs, err := s.cnrSource.List(ctx) if err != nil { s.log.Error(ctx, logs.TreeCouldNotFetchContainers, zap.Error(err)) s.metrics.AddSyncDuration(time.Since(start), false) From b0ef737a74d9e0b789be24db0830fa927e7078e0 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 16:58:21 +0300 Subject: [PATCH 17/94] [#1689] linter: Fix testifylint warning: 'len: use require.Len' Change-Id: I7a08f09c169ac237647dcb20b0737f1c51c441ad Signed-off-by: Alexander Chuprov --- .../blobstor/internal/blobstortest/iterate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index c11d0888b..d54c54f59 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -50,7 +50,7 @@ func runTestNormalHandler(t *testing.T, s common.Storage, objects []objectDesc) _, err := s.Iterate(context.Background(), iterPrm) require.NoError(t, err) - require.Equal(t, len(objects), len(seen)) + require.Len(t, objects, len(seen)) for i := range objects { d, ok := seen[objects[i].addr.String()] require.True(t, ok) From f4696e8964786dd5489fdce1456eccab010d02b8 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:03:19 +0300 Subject: [PATCH 18/94] [#1689] linter: Fix staticcheck warning: 'Use fmt.Fprintf(...) instead of WriteString(fmt.Sprintf(...))' Change-Id: I253ab717885cb01b4a2e471147e883ee351be277 Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/control/evacuation.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 8032bf09a..b8d7eb046 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -296,7 +296,7 @@ func appendEstimation(sb *strings.Builder, resp *control.GetShardEvacuationStatu leftSeconds := avgObjEvacuationTimeSeconds * objectsLeft leftMinutes := int(leftSeconds / 60) - sb.WriteString(fmt.Sprintf(" Estimated time left: %d minutes.", leftMinutes)) + fmt.Fprintf(sb, " Estimated time left: %d minutes.", leftMinutes) } func appendDuration(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { @@ -305,20 +305,20 @@ func appendDuration(sb *strings.Builder, resp *control.GetShardEvacuationStatusR hour := int(duration.Seconds() / 3600) minute := int(duration.Seconds()/60) % 60 second := int(duration.Seconds()) % 60 - sb.WriteString(fmt.Sprintf(" Duration: %02d:%02d:%02d.", hour, minute, second)) + fmt.Fprintf(sb, " Duration: %02d:%02d:%02d.", hour, minute, second) } } func appendStartedAt(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { if resp.GetBody().GetStartedAt() != nil { startedAt := time.Unix(resp.GetBody().GetStartedAt().GetValue(), 0).UTC() - sb.WriteString(fmt.Sprintf(" Started at: %s UTC.", startedAt.Format(time.RFC3339))) + fmt.Fprintf(sb, " Started at: %s UTC.", startedAt.Format(time.RFC3339)) } } func appendError(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { if len(resp.GetBody().GetErrorMessage()) > 0 { - sb.WriteString(fmt.Sprintf(" Error: %s.", resp.GetBody().GetErrorMessage())) + fmt.Fprintf(sb, " Error: %s.", resp.GetBody().GetErrorMessage()) } } @@ -332,7 +332,7 @@ func appendStatus(sb *strings.Builder, resp *control.GetShardEvacuationStatusRes default: status = "undefined" } - sb.WriteString(fmt.Sprintf(" Status: %s.", status)) + fmt.Fprintf(sb, " Status: %s.", status) } func appendShardIDs(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { @@ -350,14 +350,14 @@ func appendShardIDs(sb *strings.Builder, resp *control.GetShardEvacuationStatusR } func appendCounts(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { - sb.WriteString(fmt.Sprintf(" Evacuated %d objects out of %d, failed to evacuate: %d, skipped: %d; evacuated %d trees out of %d, failed to evacuate: %d.", + fmt.Fprintf(sb, " Evacuated %d objects out of %d, failed to evacuate: %d, skipped: %d; evacuated %d trees out of %d, failed to evacuate: %d.", resp.GetBody().GetEvacuatedObjects(), resp.GetBody().GetTotalObjects(), resp.GetBody().GetFailedObjects(), resp.GetBody().GetSkippedObjects(), resp.GetBody().GetEvacuatedTrees(), resp.GetBody().GetTotalTrees(), - resp.GetBody().GetFailedTrees())) + resp.GetBody().GetFailedTrees()) } func initControlEvacuationShardCmd() { From 4c03561aa20c2bbc2e311c9438fd7979ff1d96db Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 7 Apr 2025 18:48:36 +0300 Subject: [PATCH 19/94] [#1693] internal/assert: Add `False` and `NoError` checks Change-Id: Ib3ab1671eeff8e8917673513477f158cadbb4287 Signed-off-by: Ekaterina Lebedeva --- internal/assert/cond.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/assert/cond.go b/internal/assert/cond.go index 701036fa8..c6a034f94 100644 --- a/internal/assert/cond.go +++ b/internal/assert/cond.go @@ -1,9 +1,25 @@ package assert -import "strings" +import ( + "fmt" + "strings" +) func True(cond bool, details ...string) { if !cond { panic(strings.Join(details, " ")) } } + +func False(cond bool, details ...string) { + if cond { + panic(strings.Join(details, " ")) + } +} + +func NoError(err error, details ...string) { + if err != nil { + content := fmt.Sprintf("BUG: %v: %s", err, strings.Join(details, " ")) + panic(content) + } +} From 0e1b01b15ff21c07d809dbed00f23c6b2377bf4e Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Wed, 2 Apr 2025 19:06:10 +0300 Subject: [PATCH 20/94] [#1693] cli/adm: Replace conditional panics with asserts Change-Id: I3a46f7ac6d9e4ff51bb490e6fcfc07957418f1a7 Signed-off-by: Ekaterina Lebedeva --- .../internal/modules/morph/balance/balance.go | 9 +++----- .../modules/morph/container/container.go | 5 ++-- .../internal/modules/morph/contract/deploy.go | 10 ++++---- .../modules/morph/contract/dump_hashes.go | 9 +++----- .../modules/morph/helper/initialize_ctx.go | 23 ++++++++----------- .../modules/morph/helper/local_client.go | 9 +++----- .../morph/initialize/initialize_nns.go | 5 ++-- .../morph/initialize/initialize_register.go | 5 ++-- 8 files changed, 28 insertions(+), 47 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go index be42f2aa5..23dba14f4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -161,9 +162,7 @@ func printAlphabetContractBalances(cmd *cobra.Command, c helper.Client, inv *inv helper.GetAlphabetNNSDomain(i), int64(nns.TXT)) } - if w.Err != nil { - panic(w.Err) - } + assert.NoError(w.Err) alphaRes, err := c.InvokeScript(w.Bytes(), nil) if err != nil { @@ -226,9 +225,7 @@ func fetchBalances(c *invoker.Invoker, gasHash util.Uint160, accounts []accBalan for i := range accounts { emit.AppCall(w.BinWriter, gasHash, "balanceOf", callflag.ReadStates, accounts[i].scriptHash) } - if w.Err != nil { - panic(w.Err) - } + assert.NoError(w.Err) res, err := c.Run(w.Bytes()) if err != nil || res.State != vmstate.Halt.String() || len(res.Stack) != len(accounts) { diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index e72dc15e9..79685f111 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" "github.com/nspcc-dev/neo-go/pkg/io" @@ -235,9 +236,7 @@ func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd putContainer(bw, ch, cnt) - if bw.Err != nil { - panic(bw.Err) - } + assert.NoError(bw.Err) if err := wCtx.SendConsensusTx(bw.Bytes()); err != nil { return err diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go index 5adb480da..543b5fcb3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "github.com/nspcc-dev/neo-go/cli/cmdargs" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -120,9 +121,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { } } - if writer.Err != nil { - panic(fmt.Errorf("BUG: can't create deployment script: %w", writer.Err)) - } + assert.NoError(writer.Err, "can't create deployment script") if err := c.SendCommitteeTx(writer.Bytes(), false); err != nil { return err @@ -173,9 +172,8 @@ func registerNNS(nnsCs *state.Contract, c *helper.InitializeContext, zone string domain, int64(nns.TXT), address.Uint160ToString(cs.Hash)) } - if bw.Err != nil { - panic(fmt.Errorf("BUG: can't create deployment script: %w", writer.Err)) - } else if bw.Len() != start { + assert.NoError(bw.Err, "can't create deployment script") + if bw.Len() != start { writer.WriteBytes(bw.Bytes()) emit.Opcodes(writer.BinWriter, opcode.LDSFLD0, opcode.PUSH1, opcode.PACK) emit.AppCallNoArgs(writer.BinWriter, nnsCs.Hash, "setPrice", callflag.All) diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index 7630a226e..fde58fd2b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -236,9 +237,7 @@ func fillContractVersion(cmd *cobra.Command, c helper.Client, infos []contractDu } else { sub.Reset() emit.AppCall(sub.BinWriter, infos[i].hash, "version", callflag.NoneFlag) - if sub.Err != nil { - panic(fmt.Errorf("BUG: can't create version script: %w", bw.Err)) - } + assert.NoError(sub.Err, "can't create version script") script := sub.Bytes() emit.Instruction(bw.BinWriter, opcode.TRY, []byte{byte(3 + len(script) + 2), 0}) @@ -248,9 +247,7 @@ func fillContractVersion(cmd *cobra.Command, c helper.Client, infos []contractDu } } emit.Opcodes(bw.BinWriter, opcode.NOP) // for the last ENDTRY target - if bw.Err != nil { - panic(fmt.Errorf("BUG: can't create version script: %w", bw.Err)) - } + assert.NoError(bw.Err, "can't create version script") res, err := c.InvokeScript(bw.Bytes(), nil) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go index 27052697f..da5ffedae 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -375,9 +376,7 @@ func (c *InitializeContext) sendMultiTx(script []byte, tryGroup bool, withConsen } act, err = actor.New(c.Client, signers) } else { - if withConsensus { - panic("BUG: should never happen") - } + assert.False(withConsensus, "BUG: should never happen") act, err = c.CommitteeAct, nil } if err != nil { @@ -411,11 +410,9 @@ func (c *InitializeContext) MultiSignAndSend(tx *transaction.Transaction, accTyp func (c *InitializeContext) MultiSign(tx *transaction.Transaction, accType string) error { version, err := c.Client.GetVersion() - if err != nil { - // error appears only if client - // has not been initialized - panic(err) - } + // error appears only if client + // has not been initialized + assert.NoError(err) network := version.Protocol.Network // Use parameter context to avoid dealing with signature order. @@ -447,12 +444,12 @@ func (c *InitializeContext) MultiSign(tx *transaction.Transaction, accType strin for i := range tx.Signers { if tx.Signers[i].Account == h { + assert.True(i <= len(tx.Scripts), "BUG: invalid signing order") if i < len(tx.Scripts) { tx.Scripts[i] = *w - } else if i == len(tx.Scripts) { + } + if i == len(tx.Scripts) { tx.Scripts = append(tx.Scripts, *w) - } else { - panic("BUG: invalid signing order") } return nil } @@ -510,9 +507,7 @@ func (c *InitializeContext) NNSRegisterDomainScript(nnsHash, expectedHash util.U int64(constants.DefaultExpirationTime), constants.NNSTtlDefVal) emit.Opcodes(bw.BinWriter, opcode.ASSERT) - if bw.Err != nil { - panic(bw.Err) - } + assert.NoError(bw.Err) return bw.Bytes(), false, nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go index d0a05d5c7..46611c177 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/core" @@ -316,9 +317,7 @@ func (l *LocalClient) SendRawTransaction(tx *transaction.Transaction) (util.Uint func (l *LocalClient) putTransactions() error { // 1. Prepare new block. lastBlock, err := l.bc.GetBlock(l.bc.CurrentBlockHash()) - if err != nil { - panic(err) - } + assert.NoError(err) defer func() { l.transactions = l.transactions[:0] }() b := &block.Block{ @@ -359,9 +358,7 @@ func InvokeFunction(c Client, h util.Uint160, method string, parameters []any, s w := io.NewBufBinWriter() emit.Array(w.BinWriter, parameters...) emit.AppCallNoArgs(w.BinWriter, h, method, callflag.All) - if w.Err != nil { - panic(fmt.Sprintf("BUG: invalid parameters for '%s': %v", method, w.Err)) - } + assert.True(w.Err == nil, fmt.Sprintf("BUG: invalid parameters for '%s': %v", method, w.Err)) return c.InvokeScript(w.Bytes(), signers) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go index e127ca545..176356378 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -111,9 +112,7 @@ func wrapRegisterScriptWithPrice(w *io.BufBinWriter, nnsHash util.Uint160, s []b emit.Opcodes(w.BinWriter, opcode.LDSFLD0, opcode.PUSH1, opcode.PACK) emit.AppCallNoArgs(w.BinWriter, nnsHash, "setPrice", callflag.All) - if w.Err != nil { - panic(fmt.Errorf("BUG: can't wrap register script: %w", w.Err)) - } + assert.NoError(w.Err, "can't wrap register script") } func nnsRegisterDomain(c *helper.InitializeContext, nnsHash, expectedHash util.Uint160, domain string) error { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index 46e6621d2..7b7597d91 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -39,9 +40,7 @@ func registerCandidateRange(c *helper.InitializeContext, start, end int) error { emit.Opcodes(w.BinWriter, opcode.ASSERT) } emit.AppCall(w.BinWriter, neo.Hash, "setRegisterPrice", callflag.States, regPrice) - if w.Err != nil { - panic(fmt.Sprintf("BUG: %v", w.Err)) - } + assert.NoError(w.Err) signers := []actor.SignerAccount{{ Signer: c.GetSigner(false, c.CommitteeAcc), From 766d9ec46b7b7da10dbbd4bf257db6f5f799a968 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 7 Apr 2025 19:25:57 +0300 Subject: [PATCH 21/94] [#1693] cli/lens: Replace conditional panics with asserts Change-Id: Id827da0cd9eef66efd806be6c9bc61044175a971 Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-lens/internal/schema/common/schema.go | 8 +++----- cmd/frostfs-lens/internal/tui/records.go | 5 ++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-lens/internal/schema/common/schema.go b/cmd/frostfs-lens/internal/schema/common/schema.go index 9bad19032..077a68785 100644 --- a/cmd/frostfs-lens/internal/schema/common/schema.go +++ b/cmd/frostfs-lens/internal/schema/common/schema.go @@ -3,6 +3,8 @@ package common import ( "errors" "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" ) type FilterResult byte @@ -71,11 +73,7 @@ func (fp FallbackParser) ToParser() Parser { func (p Parser) ToFallbackParser() FallbackParser { return func(key, value []byte) (SchemaEntry, Parser) { entry, next, err := p(key, value) - if err != nil { - panic(fmt.Errorf( - "couldn't use that parser as a fallback parser, it returned an error: %w", err, - )) - } + assert.NoError(err, "couldn't use that parser as a fallback parser") return entry, next } } diff --git a/cmd/frostfs-lens/internal/tui/records.go b/cmd/frostfs-lens/internal/tui/records.go index 5f61df884..a4d392ab3 100644 --- a/cmd/frostfs-lens/internal/tui/records.go +++ b/cmd/frostfs-lens/internal/tui/records.go @@ -8,6 +8,7 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "github.com/gdamore/tcell/v2" "github.com/rivo/tview" ) @@ -94,9 +95,7 @@ func (v *RecordsView) Mount(ctx context.Context) error { } func (v *RecordsView) Unmount() { - if v.onUnmount == nil { - panic("try to unmount not mounted component") - } + assert.False(v.onUnmount == nil, "try to unmount not mounted component") v.onUnmount() v.onUnmount = nil } From 17cba3387e926c50ae47a224af47f3f6668134f4 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 8 Apr 2025 19:57:43 +0300 Subject: [PATCH 22/94] [#1332] cli/playground: Prevent prompt artifacts by writing to 'readline' stdout Change-Id: I1c3cbb0b762f29c0995d3f6fc79bae5246ee7bc3 Signed-off-by: Alexander Chuprov --- .../modules/container/policy_playground.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index dcd755510..825e1d222 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -19,8 +19,9 @@ import ( ) type policyPlaygroundREPL struct { - cmd *cobra.Command - nodes map[string]netmap.NodeInfo + cmd *cobra.Command + nodes map[string]netmap.NodeInfo + console *readline.Instance } func newPolicyPlaygroundREPL(cmd *cobra.Command) *policyPlaygroundREPL { @@ -40,7 +41,7 @@ func (repl *policyPlaygroundREPL) handleLs(args []string) error { node.IterateAttributes(func(k, v string) { attrs = append(attrs, fmt.Sprintf("%s:%q", k, v)) }) - fmt.Printf("\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) + fmt.Fprintf(repl.console, "\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) i++ } return nil @@ -147,7 +148,7 @@ func (repl *policyPlaygroundREPL) handleEval(args []string) error { for _, node := range ns { ids = append(ids, hex.EncodeToString(node.PublicKey())) } - fmt.Printf("\t%2d: %v\n", i+1, ids) + fmt.Fprintf(repl.console, "\t%2d: %v\n", i+1, ids) } return nil @@ -208,6 +209,7 @@ func (repl *policyPlaygroundREPL) run() error { if err != nil { return fmt.Errorf("error initializing readline: %w", err) } + repl.console = rl defer rl.Close() var exit bool @@ -232,10 +234,10 @@ func (repl *policyPlaygroundREPL) run() error { cmd := parts[0] if handler, exists := cmdHandlers[cmd]; exists { if err := handler(parts[1:]); err != nil { - fmt.Printf("error: %v\n", err) + fmt.Fprintf(repl.console, "error: %v\n", err) } } else { - fmt.Printf("error: unknown command %q\n", cmd) + fmt.Fprintf(repl.console, "error: unknown command %q\n", cmd) } } } From 46fd5e17b2366c22dba259d1349c6e3fa8662cb6 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 8 Apr 2025 20:23:22 +0300 Subject: [PATCH 23/94] [#1332] cli/playground: Add help Change-Id: I6160cfddf427b161619e4b96ceec8396b75c4d08 Signed-off-by: Alexander Chuprov --- .../modules/container/policy_playground.go | 119 ++++++++++++++++-- 1 file changed, 109 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 825e1d222..f747b3252 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -5,7 +5,9 @@ import ( "encoding/json" "errors" "fmt" + "maps" "os" + "slices" "strings" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" @@ -154,6 +156,23 @@ func (repl *policyPlaygroundREPL) handleEval(args []string) error { return nil } +func (repl *policyPlaygroundREPL) handleHelp(args []string) error { + if len(args) != 0 { + if _, ok := commands[args[0]]; !ok { + return fmt.Errorf("unknown command: %q", args[0]) + } + fmt.Fprintln(repl.console, commands[args[0]].usage) + return nil + } + + commandList := slices.Collect(maps.Keys(commands)) + slices.Sort(commandList) + for _, command := range commandList { + fmt.Fprintf(repl.console, "%s: %s\n", command, commands[command].descriprion) + } + return nil +} + func (repl *policyPlaygroundREPL) netMap() netmap.NetMap { var nm netmap.NetMap var nodes []netmap.NodeInfo @@ -164,15 +183,82 @@ func (repl *policyPlaygroundREPL) netMap() netmap.NetMap { return nm } -var policyPlaygroundCompleter = readline.NewPrefixCompleter( - readline.PcItem("list"), - readline.PcItem("ls"), - readline.PcItem("add"), - readline.PcItem("load"), - readline.PcItem("remove"), - readline.PcItem("rm"), - readline.PcItem("eval"), -) +type commandDescription struct { + descriprion string + usage string +} + +var commands = map[string]commandDescription{ + "list": { + descriprion: "Display all nodes in the netmap", + usage: `Display all nodes in the netmap +Example of usage: + list + 1: id=03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae attrs={Continent:"Europe" Country:"Poland"} + 2: id=02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3 attrs={Continent:"Antarctica" Country:"Heard Island"} +`, + }, + + "ls": { + descriprion: "Display all nodes in the netmap", + usage: `Display all nodes in the netmap +Example of usage: + ls + 1: id=03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae attrs={Continent:"Europe" Country:"Poland"} + 2: id=02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3 attrs={Continent:"Antarctica" Country:"Heard Island"} +`, + }, + + "add": { + descriprion: "Add a new node: add attr=value", + usage: `Add a new node +Example of usage: + add 03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae continent:Europe country:Poland`, + }, + + "load": { + descriprion: "Load netmap from file: load ", + usage: `Load netmap from file +Example of usage: + load "netmap.json" +File format (netmap.json): +{ + "03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae": { + "continent": "Europe", + "country": "Poland" + }, + "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3": { + "continent": "Antarctica", + "country": "Heard Island" + } +}`, + }, + + "remove": { + descriprion: "Remove a node: remove ", + usage: `Remove a node +Example of usage: + remove 03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae`, + }, + + "rm": { + descriprion: "Remove a node: rm ", + usage: `Remove a node +Example of usage: + rm 03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae`, + }, + + "eval": { + descriprion: "Evaluate a policy: eval ", + usage: `Evaluate a policy +Example of usage: + eval REP 2`, + }, + + "help": { + descriprion: "Show available commands", + }, +} func (repl *policyPlaygroundREPL) run() error { if len(viper.GetString(commonflags.RPC)) > 0 { @@ -199,12 +285,25 @@ func (repl *policyPlaygroundREPL) run() error { "remove": repl.handleRemove, "rm": repl.handleRemove, "eval": repl.handleEval, + "help": repl.handleHelp, } + var cfgCompleter []readline.PrefixCompleterInterface + var helpSubItems []readline.PrefixCompleterInterface + + for name := range commands { + if name != "help" { + cfgCompleter = append(cfgCompleter, readline.PcItem(name)) + helpSubItems = append(helpSubItems, readline.PcItem(name)) + } + } + + cfgCompleter = append(cfgCompleter, readline.PcItem("help", helpSubItems...)) + completer := readline.NewPrefixCompleter(cfgCompleter...) rl, err := readline.NewEx(&readline.Config{ Prompt: "> ", InterruptPrompt: "^C", - AutoComplete: policyPlaygroundCompleter, + AutoComplete: completer, }) if err != nil { return fmt.Errorf("error initializing readline: %w", err) From faec499b38a8ff06bc5c38fa31b4e75139d09d7d Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:19:34 +0300 Subject: [PATCH 24/94] [#1689] linter: Fix staticcheck warning: 'variable naming format' Change-Id: I8f8b63a6a5f9b6feb7c91f70fe8ac092575b145c Signed-off-by: Alexander Chuprov --- .../blobstor/blobovniczatree/manager.go | 26 +++++++++---------- .../blobstor/blobovniczatree/rebuild.go | 4 +-- pkg/local_object_storage/pilorama/inmemory.go | 4 +-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index f2f9509ad..6438f715b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -141,8 +141,8 @@ func (b *sharedDB) SystemPath() string { return b.path } -// levelDbManager stores pointers of the sharedDB's for the leaf directory of the blobovnicza tree. -type levelDbManager struct { +// levelDBManager stores pointers of the sharedDB's for the leaf directory of the blobovnicza tree. +type levelDBManager struct { dbMtx *sync.RWMutex databases map[uint64]*sharedDB @@ -157,8 +157,8 @@ type levelDbManager struct { func newLevelDBManager(options []blobovnicza.Option, rootPath string, lvlPath string, readOnly bool, metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlag *atomic.Bool, log *logger.Logger, -) *levelDbManager { - result := &levelDbManager{ +) *levelDBManager { + result := &levelDBManager{ databases: make(map[uint64]*sharedDB), dbMtx: &sync.RWMutex{}, @@ -173,7 +173,7 @@ func newLevelDBManager(options []blobovnicza.Option, rootPath string, lvlPath st return result } -func (m *levelDbManager) GetByIndex(idx uint64) *sharedDB { +func (m *levelDBManager) GetByIndex(idx uint64) *sharedDB { res := m.getDBIfExists(idx) if res != nil { return res @@ -181,14 +181,14 @@ func (m *levelDbManager) GetByIndex(idx uint64) *sharedDB { return m.getOrCreateDB(idx) } -func (m *levelDbManager) getDBIfExists(idx uint64) *sharedDB { +func (m *levelDBManager) getDBIfExists(idx uint64) *sharedDB { m.dbMtx.RLock() defer m.dbMtx.RUnlock() return m.databases[idx] } -func (m *levelDbManager) getOrCreateDB(idx uint64) *sharedDB { +func (m *levelDBManager) getOrCreateDB(idx uint64) *sharedDB { m.dbMtx.Lock() defer m.dbMtx.Unlock() @@ -202,7 +202,7 @@ func (m *levelDbManager) getOrCreateDB(idx uint64) *sharedDB { return db } -func (m *levelDbManager) hasAnyDB() bool { +func (m *levelDBManager) hasAnyDB() bool { m.dbMtx.RLock() defer m.dbMtx.RUnlock() @@ -213,7 +213,7 @@ func (m *levelDbManager) hasAnyDB() bool { // // The blobovnicza opens at the first request, closes after the last request. type dbManager struct { - levelToManager map[string]*levelDbManager + levelToManager map[string]*levelDBManager levelToManagerGuard *sync.RWMutex closedFlag *atomic.Bool dbCounter *openDBCounter @@ -231,7 +231,7 @@ func newDBManager(rootPath string, options []blobovnicza.Option, readOnly bool, options: options, readOnly: readOnly, metrics: metrics, - levelToManager: make(map[string]*levelDbManager), + levelToManager: make(map[string]*levelDBManager), levelToManagerGuard: &sync.RWMutex{}, log: log, closedFlag: &atomic.Bool{}, @@ -266,7 +266,7 @@ func (m *dbManager) Close() { m.dbCounter.WaitUntilAllClosed() } -func (m *dbManager) getLevelManager(lvlPath string) *levelDbManager { +func (m *dbManager) getLevelManager(lvlPath string) *levelDBManager { result := m.getLevelManagerIfExists(lvlPath) if result != nil { return result @@ -274,14 +274,14 @@ func (m *dbManager) getLevelManager(lvlPath string) *levelDbManager { return m.getOrCreateLevelManager(lvlPath) } -func (m *dbManager) getLevelManagerIfExists(lvlPath string) *levelDbManager { +func (m *dbManager) getLevelManagerIfExists(lvlPath string) *levelDBManager { m.levelToManagerGuard.RLock() defer m.levelToManagerGuard.RUnlock() return m.levelToManager[lvlPath] } -func (m *dbManager) getOrCreateLevelManager(lvlPath string) *levelDbManager { +func (m *dbManager) getOrCreateLevelManager(lvlPath string) *levelDBManager { m.levelToManagerGuard.Lock() defer m.levelToManagerGuard.Unlock() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index 7ef3317fd..d2eef2074 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -328,7 +328,7 @@ func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobo return nil } -func (b *Blobovniczas) dropDB(ctx context.Context, path string, shDb *sharedDB) (bool, error) { +func (b *Blobovniczas) dropDB(ctx context.Context, path string, shDB *sharedDB) (bool, error) { select { case <-ctx.Done(): return false, ctx.Err() @@ -341,7 +341,7 @@ func (b *Blobovniczas) dropDB(ctx context.Context, path string, shDb *sharedDB) b.dbFilesGuard.Lock() defer b.dbFilesGuard.Unlock() - if err := shDb.CloseAndRemoveFile(ctx); err != nil { + if err := shDB.CloseAndRemoveFile(ctx); err != nil { return false, err } b.commondbManager.CleanResources(path) diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index f74d12a1b..28b7faec8 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -192,7 +192,7 @@ func (t tree) getByPath(attr string, path []string, latest bool) []Node { } var nodes []Node - var lastTs Timestamp + var lastTS Timestamp children := t.getChildren(curNode) for i := range children { @@ -200,7 +200,7 @@ func (t tree) getByPath(attr string, path []string, latest bool) []Node { fileName := string(info.Meta.GetAttr(attr)) if fileName == path[len(path)-1] { if latest { - if info.Meta.Time >= lastTs { + if info.Meta.Time >= lastTS { nodes = append(nodes[:0], children[i]) } } else { From c274bbeb7c0b6957ccc5d3f507b5b5432f91ffd6 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:25:59 +0300 Subject: [PATCH 25/94] [#1689] linter: Fix staticcheck warning: 'methods on the same type should have the same receiver name' Change-Id: I25e9432987f73061c1506a184a82065e37885861 Signed-off-by: Alexander Chuprov --- pkg/services/object/get/get.go | 42 +++++++++++++++--------------- pkg/services/object/put/service.go | 4 +-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 9676fd914..3a50308c2 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -90,48 +90,48 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { return exec.err } -func (exec *request) execute(ctx context.Context) { - exec.log.Debug(ctx, logs.ServingRequest) +func (r *request) execute(ctx context.Context) { + r.log.Debug(ctx, logs.ServingRequest) // perform local operation - exec.executeLocal(ctx) + r.executeLocal(ctx) - exec.analyzeStatus(ctx, true) + r.analyzeStatus(ctx, true) } -func (exec *request) analyzeStatus(ctx context.Context, execCnr bool) { +func (r *request) analyzeStatus(ctx context.Context, execCnr bool) { // analyze local result - switch exec.status { + switch r.status { case statusOK: - exec.log.Debug(ctx, logs.OperationFinishedSuccessfully) + r.log.Debug(ctx, logs.OperationFinishedSuccessfully) case statusINHUMED: - exec.log.Debug(ctx, logs.GetRequestedObjectWasMarkedAsRemoved) + r.log.Debug(ctx, logs.GetRequestedObjectWasMarkedAsRemoved) case statusVIRTUAL: - exec.log.Debug(ctx, logs.GetRequestedObjectIsVirtual) - exec.assemble(ctx) + r.log.Debug(ctx, logs.GetRequestedObjectIsVirtual) + r.assemble(ctx) case statusOutOfRange: - exec.log.Debug(ctx, logs.GetRequestedRangeIsOutOfObjectBounds) + r.log.Debug(ctx, logs.GetRequestedRangeIsOutOfObjectBounds) case statusEC: - exec.log.Debug(ctx, logs.GetRequestedObjectIsEC) - if exec.isRaw() && execCnr { - exec.executeOnContainer(ctx) - exec.analyzeStatus(ctx, false) + r.log.Debug(ctx, logs.GetRequestedObjectIsEC) + if r.isRaw() && execCnr { + r.executeOnContainer(ctx) + r.analyzeStatus(ctx, false) } - exec.assembleEC(ctx) + r.assembleEC(ctx) default: - exec.log.Debug(ctx, logs.OperationFinishedWithError, - zap.Error(exec.err), + r.log.Debug(ctx, logs.OperationFinishedWithError, + zap.Error(r.err), ) var errAccessDenied *apistatus.ObjectAccessDenied - if execCnr && errors.As(exec.err, &errAccessDenied) { + if execCnr && errors.As(r.err, &errAccessDenied) { // Local get can't return access denied error, so this error was returned by // write to the output stream. So there is no need to try to find object on other nodes. return } if execCnr { - exec.executeOnContainer(ctx) - exec.analyzeStatus(ctx, false) + r.executeOnContainer(ctx) + r.analyzeStatus(ctx, false) } } } diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 099486b3f..7aeb5857d 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -56,8 +56,8 @@ func NewService(ks *objutil.KeyStorage, } } -func (p *Service) Put() (*Streamer, error) { +func (s *Service) Put() (*Streamer, error) { return &Streamer{ - Config: p.Config, + Config: s.Config, }, nil } From 2394ae6ce006acf85e3914fc29c3dd284be9ce3c Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:28:27 +0300 Subject: [PATCH 26/94] [#1689] linter: Fix staticcheck warning: 'could lift into loop condition' Change-Id: I4ff3cda54861d857740203d6994872998a22d5d5 Signed-off-by: Alexander Chuprov --- pkg/services/object/get/container.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pkg/services/object/get/container.go b/pkg/services/object/get/container.go index 0ee8aed53..dfb31133c 100644 --- a/pkg/services/object/get/container.go +++ b/pkg/services/object/get/container.go @@ -28,16 +28,7 @@ func (r *request) executeOnContainer(ctx context.Context) { localStatus := r.status - for { - if r.processCurrentEpoch(ctx, localStatus) { - break - } - - // check the maximum depth has been reached - if lookupDepth == 0 { - break - } - + for !r.processCurrentEpoch(ctx, localStatus) && lookupDepth != 0 { lookupDepth-- // go to the previous epoch From dfdeedfc6fe2b557fe71d58960f954f30be2c339 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:32:23 +0300 Subject: [PATCH 27/94] [#1689] linter: Fix staticcheck warning: 'could apply De Morgan's law' Change-Id: Ife03172bad7d517dc99771250c3308a9fc0916b3 Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/object/range.go | 2 +- pkg/morph/event/notary_preparator.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index be4fee4cf..6ec508ae2 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -154,7 +154,7 @@ func printECInfoErr(cmd *cobra.Command, err error) bool { if ok { toJSON, _ := cmd.Flags().GetBool(commonflags.JSON) toProto, _ := cmd.Flags().GetBool("proto") - if !(toJSON || toProto) { + if !toJSON && !toProto { cmd.PrintErrln("Object is erasure-encoded, ec information received.") } printECInfo(cmd, errECInfo.ECInfo()) diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index 40f5984a9..b11973646 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -199,8 +199,8 @@ func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { // neo-go API) // // this check prevents notary flow recursion - if !(len(nr.MainTransaction.Scripts[1].InvocationScript) == 0 || - bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript)) { // compatibility with old version + if len(nr.MainTransaction.Scripts[1].InvocationScript) != 0 && + !bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript) { // compatibility with old version return ErrTXAlreadyHandled } @@ -364,8 +364,8 @@ func (p Preparator) validateWitnesses(w []transaction.Witness, alphaKeys keys.Pu // the last one must be a placeholder for notary contract witness last := len(w) - 1 - if !(len(w[last].InvocationScript) == 0 || // https://github.com/nspcc-dev/neo-go/pull/2981 - bytes.Equal(w[last].InvocationScript, p.dummyInvocationScript)) || // compatibility with old version + if (len(w[last].InvocationScript) != 0 && // https://github.com/nspcc-dev/neo-go/pull/2981 + !bytes.Equal(w[last].InvocationScript, p.dummyInvocationScript)) || // compatibility with old version len(w[last].VerificationScript) != 0 { return errIncorrectNotaryPlaceholder } From 4f9d237042233c8ae155ae3d2de4ea3911f8ce49 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:36:54 +0300 Subject: [PATCH 28/94] [#1689] linter: Fix staticcheck warning: 'probably want to use time.Time.Equal instead' Change-Id: Idb119d3f4f167c9e42ed48633d301185589553ed Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/engine/evacuate_limiter.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index c74134500..3dd7494be 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -95,8 +95,7 @@ func (s *EvacuationState) StartedAt() *time.Time { if s == nil { return nil } - defaultTime := time.Time{} - if s.startedAt == defaultTime { + if s.startedAt.IsZero() { return nil } return &s.startedAt @@ -106,8 +105,7 @@ func (s *EvacuationState) FinishedAt() *time.Time { if s == nil { return nil } - defaultTime := time.Time{} - if s.finishedAt == defaultTime { + if s.finishedAt.IsZero() { return nil } return &s.finishedAt From fe29ed043a7d5e6e27256cf52490e1ec7ceedb40 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 8 Apr 2025 15:22:00 +0300 Subject: [PATCH 29/94] [#1689] linter: Fix staticcheck warning: 'could use tagged switch on *' Change-Id: Ia340ce1ccdd223eb87f7aefabfba62b7055f344d Signed-off-by: Alexander Chuprov --- pkg/services/object/ape/checker.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index ee46a6fe4..b96757def 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -76,9 +76,10 @@ var errMissingOID = errors.New("object ID is not set") // CheckAPE prepares an APE-request and checks if it is permitted by policies. func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { // APE check is ignored for some inter-node requests. - if prm.Role == nativeschema.PropertyValueContainerRoleContainer { + switch prm.Role { + case nativeschema.PropertyValueContainerRoleContainer: return nil - } else if prm.Role == nativeschema.PropertyValueContainerRoleIR { + case nativeschema.PropertyValueContainerRoleIR: switch prm.Method { case nativeschema.MethodGetObject, nativeschema.MethodHeadObject, From aed84b567caac5b2825888621ca170ee9dae0974 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 8 Apr 2025 15:23:51 +0300 Subject: [PATCH 30/94] [#1689] linter: Bump 'golangci-lint' to v2.0.2 Change-Id: Ib546af43845014785f0debce429a37d62e616539 Signed-off-by: Alexander Chuprov --- .golangci.yml | 179 ++++++++++++++++++++++++++------------------------ Makefile | 4 +- 2 files changed, 94 insertions(+), 89 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 18de49425..3ac4eb651 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,98 +1,103 @@ -# This file contains all available configuration options -# with their default values. - -# options for analysis running +version: "2" run: - # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 20m - - # include test files or not, default is true tests: false - -# output configuration options output: - # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" formats: - - format: tab - -# all available settings of specific linters -linters-settings: - exhaustive: - # indicates that switch statements are to be considered exhaustive if a - # 'default' case is present, even if all enum members aren't listed in the - # switch - default-signifies-exhaustive: true - gci: - sections: - - standard - - default - custom-order: true - staticcheck: - checks: ["all", "-SA1019"] # TODO Enable SA1019 after deprecated warning are fixed. - funlen: - lines: 80 # default 60 - statements: 60 # default 40 - gocognit: - min-complexity: 40 # default 30 - importas: - no-unaliased: true - no-extra-aliases: false - alias: - pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object - alias: objectSDK - unused: - field-writes-are-uses: false - exported-fields-are-used: false - local-variables-are-used: false - custom: - truecloudlab-linters: - path: bin/linters/external_linters.so - original-url: git.frostfs.info/TrueCloudLab/linters.git - settings: - noliteral: - target-methods : ["reportFlushError", "reportError"] - disable-packages: ["codes", "err", "res","exec"] - constants-package: "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - + tab: + path: stdout + colors: false linters: + default: none enable: - # mandatory linters - - revive - - predeclared - - # some default golangci-lint linters - - errcheck - - gosimple - - godot - - ineffassign - - staticcheck - - typecheck - - unused - - # extra linters - bidichk - - durationcheck - - exhaustive + - containedctx + - contextcheck - copyloopvar + - durationcheck + - errcheck + - exhaustive + - funlen + - gocognit + - godot + - importas + - ineffassign + - intrange + - misspell + - perfsprint + - predeclared + - protogetter + - reassign + - revive + - staticcheck + - testifylint + - truecloudlab-linters + - unconvert + - unparam + - unused + - usetesting + - whitespace + settings: + exhaustive: + default-signifies-exhaustive: true + funlen: + lines: 80 + statements: 60 + gocognit: + min-complexity: 40 + importas: + alias: + - pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object + alias: objectSDK + no-unaliased: true + no-extra-aliases: false + staticcheck: + checks: + - all + - -QF1002 + unused: + field-writes-are-uses: false + exported-fields-are-used: false + local-variables-are-used: false + custom: + truecloudlab-linters: + path: bin/linters/external_linters.so + original-url: git.frostfs.info/TrueCloudLab/linters.git + settings: + noliteral: + constants-package: git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs + disable-packages: + - codes + - err + - res + - exec + target-methods: + - reportFlushError + - reportError + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: - gci - gofmt - goimports - - misspell - - predeclared - - reassign - - whitespace - - containedctx - - funlen - - gocognit - - contextcheck - - importas - - truecloudlab-linters - - perfsprint - - testifylint - - protogetter - - intrange - - unconvert - - unparam - - usetesting - disable-all: true - fast: false + settings: + gci: + sections: + - standard + - default + custom-order: true + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/Makefile b/Makefile index 5b55c9eec..575eaae6f 100755 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.23 -LINT_VERSION ?= 1.64.8 +LINT_VERSION ?= 2.0.2 TRUECLOUDLAB_LINT_VERSION ?= 0.0.10 PROTOC_VERSION ?= 25.0 PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-sdk-go) @@ -224,7 +224,7 @@ lint-install: $(BIN) @@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) @rm -rf $(TMP_DIR)/linters @rmdir $(TMP_DIR) 2>/dev/null || true - @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install -trimpath github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) + @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install -trimpath github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v$(LINT_VERSION) # Run linters lint: From f93b96c60114440128782ca9d0a228a80a501ae1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Apr 2025 12:35:34 +0300 Subject: [PATCH 31/94] [#1712] adm: Add `maintenance zombie` commands Change-Id: I1b73e561a8daad67d0a8ffc0d293cbdd09aaab6b Signed-off-by: Dmitrii Stepanov --- .../internal/modules/maintenance/root.go | 15 + .../modules/maintenance/zombie/key.go | 70 +++++ .../modules/maintenance/zombie/list.go | 31 ++ .../modules/maintenance/zombie/morph.go | 46 +++ .../modules/maintenance/zombie/quarantine.go | 154 ++++++++++ .../modules/maintenance/zombie/remove.go | 55 ++++ .../modules/maintenance/zombie/restore.go | 69 +++++ .../modules/maintenance/zombie/root.go | 125 ++++++++ .../modules/maintenance/zombie/scan.go | 281 ++++++++++++++++++ .../maintenance/zombie/storage_engine.go | 203 +++++++++++++ cmd/frostfs-adm/internal/modules/root.go | 2 + 11 files changed, 1051 insertions(+) create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/root.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/key.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/list.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/morph.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/quarantine.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/remove.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/restore.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/scan.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go diff --git a/cmd/frostfs-adm/internal/modules/maintenance/root.go b/cmd/frostfs-adm/internal/modules/maintenance/root.go new file mode 100644 index 000000000..d67b70d2a --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/root.go @@ -0,0 +1,15 @@ +package maintenance + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/maintenance/zombie" + "github.com/spf13/cobra" +) + +var RootCmd = &cobra.Command{ + Use: "maintenance", + Short: "Section for maintenance commands", +} + +func init() { + RootCmd.AddCommand(zombie.Cmd) +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/key.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/key.go new file mode 100644 index 000000000..1b66889aa --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/key.go @@ -0,0 +1,70 @@ +package zombie + +import ( + "crypto/ecdsa" + "fmt" + "os" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/nspcc-dev/neo-go/cli/flags" + "github.com/nspcc-dev/neo-go/cli/input" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/wallet" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func getPrivateKey(cmd *cobra.Command, appCfg *config.Config) *ecdsa.PrivateKey { + keyDesc := viper.GetString(walletFlag) + if keyDesc == "" { + return &nodeconfig.Key(appCfg).PrivateKey + } + data, err := os.ReadFile(keyDesc) + commonCmd.ExitOnErr(cmd, "open wallet file: %w", err) + + priv, err := keys.NewPrivateKeyFromBytes(data) + if err != nil { + w, err := wallet.NewWalletFromFile(keyDesc) + commonCmd.ExitOnErr(cmd, "provided key is incorrect, only wallet or binary key supported: %w", err) + return fromWallet(cmd, w, viper.GetString(addressFlag)) + } + return &priv.PrivateKey +} + +func fromWallet(cmd *cobra.Command, w *wallet.Wallet, addrStr string) *ecdsa.PrivateKey { + var ( + addr util.Uint160 + err error + ) + + if addrStr == "" { + addr = w.GetChangeAddress() + } else { + addr, err = flags.ParseAddress(addrStr) + commonCmd.ExitOnErr(cmd, "--address option must be specified and valid: %w", err) + } + + acc := w.GetAccount(addr) + if acc == nil { + commonCmd.ExitOnErr(cmd, "--address option must be specified and valid: %w", fmt.Errorf("can't find wallet account for %s", addrStr)) + } + + pass, err := getPassword() + commonCmd.ExitOnErr(cmd, "invalid password for the encrypted key: %w", err) + + commonCmd.ExitOnErr(cmd, "can't decrypt account: %w", acc.Decrypt(pass, keys.NEP2ScryptParams())) + + return &acc.PrivateKey().PrivateKey +} + +func getPassword() (string, error) { + // this check allows empty passwords + if viper.IsSet("password") { + return viper.GetString("password"), nil + } + + return input.ReadPassword("Enter password > ") +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/list.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/list.go new file mode 100644 index 000000000..f73f33db9 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/list.go @@ -0,0 +1,31 @@ +package zombie + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" +) + +func list(cmd *cobra.Command, _ []string) { + configFile, _ := cmd.Flags().GetString(commonflags.ConfigFlag) + configDir, _ := cmd.Flags().GetString(commonflags.ConfigDirFlag) + appCfg := config.New(configFile, configDir, config.EnvPrefix) + storageEngine := newEngine(cmd, appCfg) + q := createQuarantine(cmd, storageEngine.DumpInfo()) + var containerID *cid.ID + if cidStr, _ := cmd.Flags().GetString(cidFlag); cidStr != "" { + containerID = &cid.ID{} + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", containerID.DecodeString(cidStr)) + } + + commonCmd.ExitOnErr(cmd, "iterate over quarantine: %w", q.Iterate(cmd.Context(), func(a oid.Address) error { + if containerID != nil && a.Container() != *containerID { + return nil + } + cmd.Println(a.EncodeToString()) + return nil + })) +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/morph.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/morph.go new file mode 100644 index 000000000..cd3a64499 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/morph.go @@ -0,0 +1,46 @@ +package zombie + +import ( + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" + nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + netmapClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "github.com/spf13/cobra" +) + +func createMorphClient(cmd *cobra.Command, appCfg *config.Config) *client.Client { + addresses := morphconfig.RPCEndpoint(appCfg) + if len(addresses) == 0 { + commonCmd.ExitOnErr(cmd, "create morph client: %w", errors.New("no morph endpoints found")) + } + key := nodeconfig.Key(appCfg) + cli, err := client.New(cmd.Context(), + key, + client.WithDialTimeout(morphconfig.DialTimeout(appCfg)), + client.WithEndpoints(addresses...), + client.WithSwitchInterval(morphconfig.SwitchInterval(appCfg)), + ) + commonCmd.ExitOnErr(cmd, "create morph client: %w", err) + return cli +} + +func createContainerClient(cmd *cobra.Command, morph *client.Client) *cntClient.Client { + hs, err := morph.NNSContractAddress(client.NNSContainerContractName) + commonCmd.ExitOnErr(cmd, "resolve container contract hash: %w", err) + cc, err := cntClient.NewFromMorph(morph, hs, 0) + commonCmd.ExitOnErr(cmd, "create morph container client: %w", err) + return cc +} + +func createNetmapClient(cmd *cobra.Command, morph *client.Client) *netmapClient.Client { + hs, err := morph.NNSContractAddress(client.NNSNetmapContractName) + commonCmd.ExitOnErr(cmd, "resolve netmap contract hash: %w", err) + cli, err := netmapClient.NewFromMorph(morph, hs, 0) + commonCmd.ExitOnErr(cmd, "create morph netmap client: %w", err) + return cli +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/quarantine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/quarantine.go new file mode 100644 index 000000000..27f83aec7 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/quarantine.go @@ -0,0 +1,154 @@ +package zombie + +import ( + "context" + "fmt" + "math" + "os" + "path/filepath" + "strings" + "sync" + + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" +) + +type quarantine struct { + // mtx protects current field. + mtx sync.Mutex + current int + trees []*fstree.FSTree +} + +func createQuarantine(cmd *cobra.Command, engineInfo engine.Info) *quarantine { + var paths []string + for _, sh := range engineInfo.Shards { + var storagePaths []string + for _, st := range sh.BlobStorInfo.SubStorages { + storagePaths = append(storagePaths, st.Path) + } + if len(storagePaths) == 0 { + continue + } + paths = append(paths, filepath.Join(commonPath(storagePaths), "quarantine")) + } + q, err := newQuarantine(paths) + commonCmd.ExitOnErr(cmd, "create quarantine: %w", err) + return q +} + +func commonPath(paths []string) string { + if len(paths) == 0 { + return "" + } + if len(paths) == 1 { + return paths[0] + } + minLen := math.MaxInt + for _, p := range paths { + if len(p) < minLen { + minLen = len(p) + } + } + + var sb strings.Builder + for i := range minLen { + for _, path := range paths[1:] { + if paths[0][i] != path[i] { + return sb.String() + } + } + sb.WriteByte(paths[0][i]) + } + return sb.String() +} + +func newQuarantine(paths []string) (*quarantine, error) { + var q quarantine + for i := range paths { + f := fstree.New( + fstree.WithDepth(1), + fstree.WithDirNameLen(1), + fstree.WithPath(paths[i]), + fstree.WithPerm(os.ModePerm), + ) + if err := f.Open(mode.ComponentReadWrite); err != nil { + return nil, fmt.Errorf("open fstree %s: %w", paths[i], err) + } + if err := f.Init(); err != nil { + return nil, fmt.Errorf("init fstree %s: %w", paths[i], err) + } + q.trees = append(q.trees, f) + } + return &q, nil +} + +func (q *quarantine) Get(ctx context.Context, a oid.Address) (*objectSDK.Object, error) { + for i := range q.trees { + res, err := q.trees[i].Get(ctx, common.GetPrm{Address: a}) + if err != nil { + continue + } + return res.Object, nil + } + return nil, &apistatus.ObjectNotFound{} +} + +func (q *quarantine) Delete(ctx context.Context, a oid.Address) error { + for i := range q.trees { + _, err := q.trees[i].Delete(ctx, common.DeletePrm{Address: a}) + if err != nil { + continue + } + return nil + } + return &apistatus.ObjectNotFound{} +} + +func (q *quarantine) Put(ctx context.Context, obj *objectSDK.Object) error { + data, err := obj.Marshal() + if err != nil { + return err + } + + var prm common.PutPrm + prm.Address = objectcore.AddressOf(obj) + prm.Object = obj + prm.RawData = data + + q.mtx.Lock() + current := q.current + q.current = (q.current + 1) % len(q.trees) + q.mtx.Unlock() + + _, err = q.trees[current].Put(ctx, prm) + return err +} + +func (q *quarantine) Iterate(ctx context.Context, f func(oid.Address) error) error { + var prm common.IteratePrm + prm.Handler = func(elem common.IterationElement) error { + return f(elem.Address) + } + for i := range q.trees { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + _, err := q.trees[i].Iterate(ctx, prm) + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/remove.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/remove.go new file mode 100644 index 000000000..0b8f2f172 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/remove.go @@ -0,0 +1,55 @@ +package zombie + +import ( + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" +) + +func remove(cmd *cobra.Command, _ []string) { + configFile, _ := cmd.Flags().GetString(commonflags.ConfigFlag) + configDir, _ := cmd.Flags().GetString(commonflags.ConfigDirFlag) + appCfg := config.New(configFile, configDir, config.EnvPrefix) + storageEngine := newEngine(cmd, appCfg) + q := createQuarantine(cmd, storageEngine.DumpInfo()) + + var containerID cid.ID + cidStr, _ := cmd.Flags().GetString(cidFlag) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", containerID.DecodeString(cidStr)) + + var objectID *oid.ID + oidStr, _ := cmd.Flags().GetString(oidFlag) + if oidStr != "" { + objectID = &oid.ID{} + commonCmd.ExitOnErr(cmd, "decode object ID string: %w", objectID.DecodeString(oidStr)) + } + + if objectID != nil { + var addr oid.Address + addr.SetContainer(containerID) + addr.SetObject(*objectID) + removeObject(cmd, q, addr) + } else { + commonCmd.ExitOnErr(cmd, "iterate over quarantine: %w", q.Iterate(cmd.Context(), func(addr oid.Address) error { + if addr.Container() != containerID { + return nil + } + removeObject(cmd, q, addr) + return nil + })) + } +} + +func removeObject(cmd *cobra.Command, q *quarantine, addr oid.Address) { + err := q.Delete(cmd.Context(), addr) + if errors.Is(err, new(apistatus.ObjectNotFound)) { + return + } + commonCmd.ExitOnErr(cmd, "remove object from quarantine: %w", err) +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/restore.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/restore.go new file mode 100644 index 000000000..f179c7c2d --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/restore.go @@ -0,0 +1,69 @@ +package zombie + +import ( + "crypto/sha256" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" +) + +func restore(cmd *cobra.Command, _ []string) { + configFile, _ := cmd.Flags().GetString(commonflags.ConfigFlag) + configDir, _ := cmd.Flags().GetString(commonflags.ConfigDirFlag) + appCfg := config.New(configFile, configDir, config.EnvPrefix) + storageEngine := newEngine(cmd, appCfg) + q := createQuarantine(cmd, storageEngine.DumpInfo()) + morphClient := createMorphClient(cmd, appCfg) + cnrCli := createContainerClient(cmd, morphClient) + + var containerID cid.ID + cidStr, _ := cmd.Flags().GetString(cidFlag) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", containerID.DecodeString(cidStr)) + + var objectID *oid.ID + oidStr, _ := cmd.Flags().GetString(oidFlag) + if oidStr != "" { + objectID = &oid.ID{} + commonCmd.ExitOnErr(cmd, "decode object ID string: %w", objectID.DecodeString(oidStr)) + } + + if objectID != nil { + var addr oid.Address + addr.SetContainer(containerID) + addr.SetObject(*objectID) + restoreObject(cmd, storageEngine, q, addr, cnrCli) + } else { + commonCmd.ExitOnErr(cmd, "iterate over quarantine: %w", q.Iterate(cmd.Context(), func(addr oid.Address) error { + if addr.Container() != containerID { + return nil + } + restoreObject(cmd, storageEngine, q, addr, cnrCli) + return nil + })) + } +} + +func restoreObject(cmd *cobra.Command, storageEngine *engine.StorageEngine, q *quarantine, addr oid.Address, cnrCli *cntClient.Client) { + obj, err := q.Get(cmd.Context(), addr) + commonCmd.ExitOnErr(cmd, "get object from quarantine: %w", err) + rawCID := make([]byte, sha256.Size) + + cid := addr.Container() + cid.Encode(rawCID) + cnr, err := cnrCli.Get(cmd.Context(), rawCID) + commonCmd.ExitOnErr(cmd, "get container: %w", err) + + putPrm := engine.PutPrm{ + Object: obj, + IsIndexedContainer: containerCore.IsIndexedContainer(cnr.Value), + } + commonCmd.ExitOnErr(cmd, "put object to storage engine: %w", storageEngine.Put(cmd.Context(), putPrm)) + commonCmd.ExitOnErr(cmd, "remove object from quarantine: %w", q.Delete(cmd.Context(), addr)) +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go new file mode 100644 index 000000000..9ef18f7f8 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go @@ -0,0 +1,125 @@ +package zombie + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + flagBatchSize = "batch-size" + flagBatchSizeUsage = "Objects iteration batch size" + cidFlag = "cid" + cidFlagUsage = "Container ID" + oidFlag = "oid" + oidFlagUsage = "Object ID" + walletFlag = "wallet" + walletFlagShorthand = "w" + walletFlagUsage = "Path to the wallet or binary key" + addressFlag = "address" + addressFlagUsage = "Address of wallet account" + moveFlag = "move" + moveFlagUsage = "Move objects from storage engine to quarantine" +) + +var ( + Cmd = &cobra.Command{ + Use: "zombie", + Short: "Zombie objects related commands", + } + scanCmd = &cobra.Command{ + Use: "scan", + Short: "Scan storage engine for zombie objects and move them to quarantine", + Long: "", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.ConfigFlag, cmd.Flags().Lookup(commonflags.ConfigFlag)) + _ = viper.BindPFlag(commonflags.ConfigDirFlag, cmd.Flags().Lookup(commonflags.ConfigDirFlag)) + _ = viper.BindPFlag(walletFlag, cmd.Flags().Lookup(walletFlag)) + _ = viper.BindPFlag(addressFlag, cmd.Flags().Lookup(addressFlag)) + _ = viper.BindPFlag(flagBatchSize, cmd.Flags().Lookup(flagBatchSize)) + _ = viper.BindPFlag(moveFlag, cmd.Flags().Lookup(moveFlag)) + }, + Run: scan, + } + listCmd = &cobra.Command{ + Use: "list", + Short: "List zombie objects from quarantine", + Long: "", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.ConfigFlag, cmd.Flags().Lookup(commonflags.ConfigFlag)) + _ = viper.BindPFlag(commonflags.ConfigDirFlag, cmd.Flags().Lookup(commonflags.ConfigDirFlag)) + _ = viper.BindPFlag(cidFlag, cmd.Flags().Lookup(cidFlag)) + }, + Run: list, + } + restoreCmd = &cobra.Command{ + Use: "restore", + Short: "Restore zombie objects from quarantine", + Long: "", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.ConfigFlag, cmd.Flags().Lookup(commonflags.ConfigFlag)) + _ = viper.BindPFlag(commonflags.ConfigDirFlag, cmd.Flags().Lookup(commonflags.ConfigDirFlag)) + _ = viper.BindPFlag(cidFlag, cmd.Flags().Lookup(cidFlag)) + _ = viper.BindPFlag(oidFlag, cmd.Flags().Lookup(oidFlag)) + }, + Run: restore, + } + removeCmd = &cobra.Command{ + Use: "remove", + Short: "Remove zombie objects from quarantine", + Long: "", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.ConfigFlag, cmd.Flags().Lookup(commonflags.ConfigFlag)) + _ = viper.BindPFlag(commonflags.ConfigDirFlag, cmd.Flags().Lookup(commonflags.ConfigDirFlag)) + _ = viper.BindPFlag(cidFlag, cmd.Flags().Lookup(cidFlag)) + _ = viper.BindPFlag(oidFlag, cmd.Flags().Lookup(oidFlag)) + }, + Run: remove, + } +) + +func init() { + initScanCmd() + initListCmd() + initRestoreCmd() + initRemoveCmd() +} + +func initScanCmd() { + Cmd.AddCommand(scanCmd) + + scanCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + scanCmd.Flags().StringP(commonflags.ConfigFlag, commonflags.ConfigFlagShorthand, "", commonflags.ConfigFlagUsage) + scanCmd.Flags().String(commonflags.ConfigDirFlag, "", commonflags.ConfigDirFlagUsage) + scanCmd.Flags().Uint32(flagBatchSize, 1000, flagBatchSizeUsage) + scanCmd.Flags().StringP(walletFlag, walletFlagShorthand, "", walletFlagUsage) + scanCmd.Flags().String(addressFlag, "", addressFlagUsage) + scanCmd.Flags().Bool(moveFlag, false, moveFlagUsage) +} + +func initListCmd() { + Cmd.AddCommand(listCmd) + + listCmd.Flags().StringP(commonflags.ConfigFlag, commonflags.ConfigFlagShorthand, "", commonflags.ConfigFlagUsage) + listCmd.Flags().String(commonflags.ConfigDirFlag, "", commonflags.ConfigDirFlagUsage) + listCmd.Flags().String(cidFlag, "", cidFlagUsage) +} + +func initRestoreCmd() { + Cmd.AddCommand(restoreCmd) + + restoreCmd.Flags().StringP(commonflags.ConfigFlag, commonflags.ConfigFlagShorthand, "", commonflags.ConfigFlagUsage) + restoreCmd.Flags().String(commonflags.ConfigDirFlag, "", commonflags.ConfigDirFlagUsage) + restoreCmd.Flags().String(cidFlag, "", cidFlagUsage) + restoreCmd.Flags().String(oidFlag, "", oidFlagUsage) +} + +func initRemoveCmd() { + Cmd.AddCommand(removeCmd) + + removeCmd.Flags().StringP(commonflags.ConfigFlag, commonflags.ConfigFlagShorthand, "", commonflags.ConfigFlagUsage) + removeCmd.Flags().String(commonflags.ConfigDirFlag, "", commonflags.ConfigDirFlagUsage) + removeCmd.Flags().String(cidFlag, "", cidFlagUsage) + removeCmd.Flags().String(oidFlag, "", oidFlagUsage) +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/scan.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/scan.go new file mode 100644 index 000000000..268ec4911 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/scan.go @@ -0,0 +1,281 @@ +package zombie + +import ( + "context" + "crypto/ecdsa" + "crypto/sha256" + "errors" + "fmt" + "sync" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + clientCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/cache" + clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +func scan(cmd *cobra.Command, _ []string) { + configFile, _ := cmd.Flags().GetString(commonflags.ConfigFlag) + configDir, _ := cmd.Flags().GetString(commonflags.ConfigDirFlag) + appCfg := config.New(configFile, configDir, config.EnvPrefix) + batchSize, _ := cmd.Flags().GetUint32(flagBatchSize) + if batchSize == 0 { + commonCmd.ExitOnErr(cmd, "invalid batch size: %w", errors.New("batch size must be positive value")) + } + move, _ := cmd.Flags().GetBool(moveFlag) + + storageEngine := newEngine(cmd, appCfg) + morphClient := createMorphClient(cmd, appCfg) + cnrCli := createContainerClient(cmd, morphClient) + nmCli := createNetmapClient(cmd, morphClient) + q := createQuarantine(cmd, storageEngine.DumpInfo()) + pk := getPrivateKey(cmd, appCfg) + + epoch, err := nmCli.Epoch(cmd.Context()) + commonCmd.ExitOnErr(cmd, "read epoch from morph: %w", err) + + nm, err := nmCli.GetNetMapByEpoch(cmd.Context(), epoch) + commonCmd.ExitOnErr(cmd, "read netmap from morph: %w", err) + + cmd.Printf("Epoch: %d\n", nm.Epoch()) + cmd.Printf("Nodes in the netmap: %d\n", len(nm.Nodes())) + + ps := &processStatus{ + statusCount: make(map[status]uint64), + } + + stopCh := make(chan struct{}) + start := time.Now() + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-cmd.Context().Done(): + return + case <-stopCh: + return + case <-tick.C: + fmt.Printf("Objects processed: %d; Time elapsed: %s\n", ps.total(), time.Since(start)) + } + } + }() + go func() { + defer wg.Done() + err = scanStorageEngine(cmd, batchSize, storageEngine, ps, appCfg, cnrCli, nmCli, q, pk, move) + close(stopCh) + }() + wg.Wait() + commonCmd.ExitOnErr(cmd, "scan storage engine for zombie objects: %w", err) + + cmd.Println() + cmd.Println("Status description:") + cmd.Println("undefined -- nothing is clear") + cmd.Println("found -- object is found in cluster") + cmd.Println("quarantine -- object is not found in cluster") + cmd.Println() + for status, count := range ps.statusCount { + cmd.Printf("Status: %s, Count: %d\n", status, count) + } +} + +type status string + +const ( + statusUndefined status = "undefined" + statusFound status = "found" + statusQuarantine status = "quarantine" +) + +func checkAddr(ctx context.Context, cnrCli *cntClient.Client, nmCli *netmap.Client, cc *cache.ClientCache, obj object.Info) (status, error) { + rawCID := make([]byte, sha256.Size) + cid := obj.Address.Container() + cid.Encode(rawCID) + + cnr, err := cnrCli.Get(ctx, rawCID) + if err != nil { + var errContainerNotFound *apistatus.ContainerNotFound + if errors.As(err, &errContainerNotFound) { + // Policer will deal with this object. + return statusFound, nil + } + return statusUndefined, fmt.Errorf("read container %s from morph: %w", cid, err) + } + nm, err := nmCli.NetMap(ctx) + if err != nil { + return statusUndefined, fmt.Errorf("read netmap from morph: %w", err) + } + + nodes, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), rawCID) + if err != nil { + // Not enough nodes, check all netmap nodes. + nodes = append([][]netmap.NodeInfo{}, nm.Nodes()) + } + + objID := obj.Address.Object() + cnrID := obj.Address.Container() + local := true + raw := false + if obj.ECInfo != nil { + objID = obj.ECInfo.ParentID + local = false + raw = true + } + prm := clientSDK.PrmObjectHead{ + ObjectID: &objID, + ContainerID: &cnrID, + Local: local, + Raw: raw, + } + + var ni clientCore.NodeInfo + for i := range nodes { + for j := range nodes[i] { + if err := clientCore.NodeInfoFromRawNetmapElement(&ni, netmapCore.Node(nodes[i][j])); err != nil { + return statusUndefined, fmt.Errorf("parse node info: %w", err) + } + c, err := cc.Get(ni) + if err != nil { + continue + } + res, err := c.ObjectHead(ctx, prm) + if err != nil { + var errECInfo *objectSDK.ECInfoError + if raw && errors.As(err, &errECInfo) { + return statusFound, nil + } + continue + } + if err := apistatus.ErrFromStatus(res.Status()); err != nil { + continue + } + return statusFound, nil + } + } + + if cnr.Value.PlacementPolicy().NumberOfReplicas() == 1 && cnr.Value.PlacementPolicy().ReplicaDescriptor(0).NumberOfObjects() == 1 { + return statusFound, nil + } + return statusQuarantine, nil +} + +func scanStorageEngine(cmd *cobra.Command, batchSize uint32, storageEngine *engine.StorageEngine, ps *processStatus, + appCfg *config.Config, cnrCli *cntClient.Client, nmCli *netmap.Client, q *quarantine, pk *ecdsa.PrivateKey, move bool, +) error { + cc := cache.NewSDKClientCache(cache.ClientCacheOpts{ + DialTimeout: apiclientconfig.DialTimeout(appCfg), + StreamTimeout: apiclientconfig.StreamTimeout(appCfg), + ReconnectTimeout: apiclientconfig.ReconnectTimeout(appCfg), + Key: pk, + AllowExternal: apiclientconfig.AllowExternal(appCfg), + }) + ctx := cmd.Context() + + var cursor *engine.Cursor + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + var prm engine.ListWithCursorPrm + prm.WithCursor(cursor) + prm.WithCount(batchSize) + + res, err := storageEngine.ListWithCursor(ctx, prm) + if err != nil { + if errors.Is(err, engine.ErrEndOfListing) { + return nil + } + return fmt.Errorf("list with cursor: %w", err) + } + + cursor = res.Cursor() + addrList := res.AddressList() + eg, egCtx := errgroup.WithContext(ctx) + eg.SetLimit(int(batchSize)) + + for i := range addrList { + addr := addrList[i] + eg.Go(func() error { + result, err := checkAddr(egCtx, cnrCli, nmCli, cc, addr) + if err != nil { + return fmt.Errorf("check object %s status: %w", addr.Address, err) + } + ps.add(result) + + if !move && result == statusQuarantine { + cmd.Println(addr) + return nil + } + + if result == statusQuarantine { + return moveToQuarantine(egCtx, storageEngine, q, addr.Address) + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return fmt.Errorf("process objects batch: %w", err) + } + } +} + +func moveToQuarantine(ctx context.Context, storageEngine *engine.StorageEngine, q *quarantine, addr oid.Address) error { + var getPrm engine.GetPrm + getPrm.WithAddress(addr) + res, err := storageEngine.Get(ctx, getPrm) + if err != nil { + return fmt.Errorf("get object %s from storage engine: %w", addr, err) + } + + if err := q.Put(ctx, res.Object()); err != nil { + return fmt.Errorf("put object %s to quarantine: %w", addr, err) + } + + var delPrm engine.DeletePrm + delPrm.WithForceRemoval() + delPrm.WithAddress(addr) + + if err = storageEngine.Delete(ctx, delPrm); err != nil { + return fmt.Errorf("delete object %s from storage engine: %w", addr, err) + } + return nil +} + +type processStatus struct { + guard sync.RWMutex + statusCount map[status]uint64 + count uint64 +} + +func (s *processStatus) add(st status) { + s.guard.Lock() + defer s.guard.Unlock() + s.statusCount[st]++ + s.count++ +} + +func (s *processStatus) total() uint64 { + s.guard.RLock() + defer s.guard.RUnlock() + return s.count +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go new file mode 100644 index 000000000..5851e049c --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go @@ -0,0 +1,203 @@ +package zombie + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + engineconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine" + shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" + blobovniczaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza" + fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "github.com/panjf2000/ants/v2" + "github.com/spf13/cobra" + "go.etcd.io/bbolt" + "go.uber.org/zap" +) + +func newEngine(cmd *cobra.Command, c *config.Config) *engine.StorageEngine { + ngOpts := storageEngineOptions(c) + shardOpts := shardOptions(cmd, c) + e := engine.New(ngOpts...) + for _, opts := range shardOpts { + _, err := e.AddShard(cmd.Context(), opts...) + commonCmd.ExitOnErr(cmd, "iterate shards from config: %w", err) + } + commonCmd.ExitOnErr(cmd, "open storage engine: %w", e.Open(cmd.Context())) + commonCmd.ExitOnErr(cmd, "init storage engine: %w", e.Init(cmd.Context())) + return e +} + +func storageEngineOptions(c *config.Config) []engine.Option { + return []engine.Option{ + engine.WithErrorThreshold(engineconfig.ShardErrorThreshold(c)), + engine.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + engine.WithLowMemoryConsumption(engineconfig.EngineLowMemoryConsumption(c)), + } +} + +func shardOptions(cmd *cobra.Command, c *config.Config) [][]shard.Option { + var result [][]shard.Option + err := engineconfig.IterateShards(c, false, func(sh *shardconfig.Config) error { + result = append(result, getShardOpts(cmd, c, sh)) + return nil + }) + commonCmd.ExitOnErr(cmd, "iterate shards from config: %w", err) + return result +} + +func getShardOpts(cmd *cobra.Command, c *config.Config, sh *shardconfig.Config) []shard.Option { + wc, wcEnabled := getWriteCacheOpts(sh) + return []shard.Option{ + shard.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + shard.WithRefillMetabase(sh.RefillMetabase()), + shard.WithRefillMetabaseWorkersCount(sh.RefillMetabaseWorkersCount()), + shard.WithMode(sh.Mode()), + shard.WithBlobStorOptions(getBlobstorOpts(cmd.Context(), sh)...), + shard.WithMetaBaseOptions(getMetabaseOpts(sh)...), + shard.WithPiloramaOptions(getPiloramaOpts(c, sh)...), + shard.WithWriteCache(wcEnabled), + shard.WithWriteCacheOptions(wc), + shard.WithRemoverBatchSize(sh.GC().RemoverBatchSize()), + shard.WithGCRemoverSleepInterval(sh.GC().RemoverSleepInterval()), + shard.WithExpiredCollectorBatchSize(sh.GC().ExpiredCollectorBatchSize()), + shard.WithExpiredCollectorWorkerCount(sh.GC().ExpiredCollectorWorkerCount()), + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + commonCmd.ExitOnErr(cmd, "init GC pool: %w", err) + return pool + }), + shard.WithLimiter(qos.NewNoopLimiter()), + } +} + +func getWriteCacheOpts(sh *shardconfig.Config) ([]writecache.Option, bool) { + if wc := sh.WriteCache(); wc != nil && wc.Enabled() { + var result []writecache.Option + result = append(result, + writecache.WithPath(wc.Path()), + writecache.WithFlushSizeLimit(wc.MaxFlushingObjectsSize()), + writecache.WithMaxObjectSize(wc.MaxObjectSize()), + writecache.WithFlushWorkersCount(wc.WorkerCount()), + writecache.WithMaxCacheSize(wc.SizeLimit()), + writecache.WithMaxCacheCount(wc.CountLimit()), + writecache.WithNoSync(wc.NoSync()), + writecache.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + writecache.WithQoSLimiter(qos.NewNoopLimiter()), + ) + return result, true + } + return nil, false +} + +func getPiloramaOpts(c *config.Config, sh *shardconfig.Config) []pilorama.Option { + var piloramaOpts []pilorama.Option + if config.BoolSafe(c.Sub("tree"), "enabled") { + pr := sh.Pilorama() + piloramaOpts = append(piloramaOpts, + pilorama.WithPath(pr.Path()), + pilorama.WithPerm(pr.Perm()), + pilorama.WithNoSync(pr.NoSync()), + pilorama.WithMaxBatchSize(pr.MaxBatchSize()), + pilorama.WithMaxBatchDelay(pr.MaxBatchDelay()), + ) + } + return piloramaOpts +} + +func getMetabaseOpts(sh *shardconfig.Config) []meta.Option { + return []meta.Option{ + meta.WithPath(sh.Metabase().Path()), + meta.WithPermissions(sh.Metabase().BoltDB().Perm()), + meta.WithMaxBatchSize(sh.Metabase().BoltDB().MaxBatchSize()), + meta.WithMaxBatchDelay(sh.Metabase().BoltDB().MaxBatchDelay()), + meta.WithBoltDBOptions(&bbolt.Options{ + Timeout: 100 * time.Millisecond, + }), + meta.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + meta.WithEpochState(&epochState{}), + } +} + +func getBlobstorOpts(ctx context.Context, sh *shardconfig.Config) []blobstor.Option { + result := []blobstor.Option{ + blobstor.WithCompressObjects(sh.Compress()), + blobstor.WithUncompressableContentTypes(sh.UncompressableContentTypes()), + blobstor.WithCompressibilityEstimate(sh.EstimateCompressibility()), + blobstor.WithCompressibilityEstimateThreshold(sh.EstimateCompressibilityThreshold()), + blobstor.WithStorages(getSubStorages(ctx, sh)), + blobstor.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + } + + return result +} + +func getSubStorages(ctx context.Context, sh *shardconfig.Config) []blobstor.SubStorage { + var ss []blobstor.SubStorage + for _, storage := range sh.BlobStor().Storages() { + switch storage.Type() { + case blobovniczatree.Type: + sub := blobovniczaconfig.From((*config.Config)(storage)) + blobTreeOpts := []blobovniczatree.Option{ + blobovniczatree.WithRootPath(storage.Path()), + blobovniczatree.WithPermissions(storage.Perm()), + blobovniczatree.WithBlobovniczaSize(sub.Size()), + blobovniczatree.WithBlobovniczaShallowDepth(sub.ShallowDepth()), + blobovniczatree.WithBlobovniczaShallowWidth(sub.ShallowWidth()), + blobovniczatree.WithOpenedCacheSize(sub.OpenedCacheSize()), + blobovniczatree.WithOpenedCacheTTL(sub.OpenedCacheTTL()), + blobovniczatree.WithOpenedCacheExpInterval(sub.OpenedCacheExpInterval()), + blobovniczatree.WithInitWorkerCount(sub.InitWorkerCount()), + blobovniczatree.WithWaitBeforeDropDB(sub.RebuildDropTimeout()), + blobovniczatree.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + blobovniczatree.WithObjectSizeLimit(sh.SmallSizeLimit()), + } + + ss = append(ss, blobstor.SubStorage{ + Storage: blobovniczatree.NewBlobovniczaTree(ctx, blobTreeOpts...), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) < sh.SmallSizeLimit() + }, + }) + case fstree.Type: + sub := fstreeconfig.From((*config.Config)(storage)) + fstreeOpts := []fstree.Option{ + fstree.WithPath(storage.Path()), + fstree.WithPerm(storage.Perm()), + fstree.WithDepth(sub.Depth()), + fstree.WithNoSync(sub.NoSync()), + fstree.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + } + + ss = append(ss, blobstor.SubStorage{ + Storage: fstree.New(fstreeOpts...), + Policy: func(_ *objectSDK.Object, _ []byte) bool { + return true + }, + }) + default: + // should never happen, that has already + // been handled: when the config was read + } + } + return ss +} + +type epochState struct{} + +func (epochState) CurrentEpoch() uint64 { + return 0 +} diff --git a/cmd/frostfs-adm/internal/modules/root.go b/cmd/frostfs-adm/internal/modules/root.go index e42204b7a..cc8225c7a 100644 --- a/cmd/frostfs-adm/internal/modules/root.go +++ b/cmd/frostfs-adm/internal/modules/root.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/maintenance" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" @@ -41,6 +42,7 @@ func init() { rootCmd.AddCommand(config.RootCmd) rootCmd.AddCommand(morph.RootCmd) rootCmd.AddCommand(metabase.RootCmd) + rootCmd.AddCommand(maintenance.RootCmd) rootCmd.AddCommand(autocomplete.Command("frostfs-adm")) rootCmd.AddCommand(gendoc.Command(rootCmd, gendoc.Options{})) From f7779adf71babe6c06a91172e3ef95c947c1f1a7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Apr 2025 15:32:23 +0300 Subject: [PATCH 32/94] [#1712] core: Extend object info string with EC header Closes #1712 Change-Id: Ief4a960f7dece3359763113270d1ff5155f3f19e Signed-off-by: Dmitrii Stepanov --- pkg/core/object/info.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/core/object/info.go b/pkg/core/object/info.go index 67c9a3188..aab12ebf9 100644 --- a/pkg/core/object/info.go +++ b/pkg/core/object/info.go @@ -13,6 +13,13 @@ type ECInfo struct { Total uint32 } +func (v *ECInfo) String() string { + if v == nil { + return "" + } + return fmt.Sprintf("parent ID: %s, index: %d, total %d", v.ParentID, v.Index, v.Total) +} + // Info groups object address with its FrostFS // object info. type Info struct { @@ -23,5 +30,5 @@ type Info struct { } func (v Info) String() string { - return fmt.Sprintf("address: %s, type: %s, is linking: %t", v.Address, v.Type, v.IsLinkingObject) + return fmt.Sprintf("address: %s, type: %s, is linking: %t, EC header: %s", v.Address, v.Type, v.IsLinkingObject, v.ECInfo) } From 6730e27ae71a97af5ca33392ce6ab5e9e3257173 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Apr 2025 16:29:48 +0300 Subject: [PATCH 33/94] [#1712] adm: Drop rpc-endpoint flag from `zombie scan` Morph addresses from config are used. Change-Id: Id99f91defbbff442c308f30d219b9824b4c871de Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go index 9ef18f7f8..c8fd9e5e5 100644 --- a/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go @@ -32,7 +32,6 @@ var ( Short: "Scan storage engine for zombie objects and move them to quarantine", Long: "", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(commonflags.ConfigFlag, cmd.Flags().Lookup(commonflags.ConfigFlag)) _ = viper.BindPFlag(commonflags.ConfigDirFlag, cmd.Flags().Lookup(commonflags.ConfigDirFlag)) _ = viper.BindPFlag(walletFlag, cmd.Flags().Lookup(walletFlag)) @@ -89,7 +88,6 @@ func init() { func initScanCmd() { Cmd.AddCommand(scanCmd) - scanCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) scanCmd.Flags().StringP(commonflags.ConfigFlag, commonflags.ConfigFlagShorthand, "", commonflags.ConfigFlagUsage) scanCmd.Flags().String(commonflags.ConfigDirFlag, "", commonflags.ConfigDirFlagUsage) scanCmd.Flags().Uint32(flagBatchSize, 1000, flagBatchSizeUsage) From dcfd89544925719560343521ceb9a792d195cb51 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 8 Apr 2025 10:21:14 +0300 Subject: [PATCH 34/94] [#1710] object: Implement `Unwrap()` for `errIncompletePut` * When sign service calls `SignResponse`, it tries to set v2 status to response by unwrapping an error to the possible depth. This wasn't applicable for `errIncompletePut` so far as it didn't implement `Unwrap()`. Thus, it wasn't able to find a correct status set in error. Change-Id: I280c1806a008176854c55f13bf8688e5736ef941 Signed-off-by: Airat Arifullin --- pkg/services/object/common/writer/distributed.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/services/object/common/writer/distributed.go b/pkg/services/object/common/writer/distributed.go index f7486eae7..fff58aca7 100644 --- a/pkg/services/object/common/writer/distributed.go +++ b/pkg/services/object/common/writer/distributed.go @@ -95,6 +95,10 @@ func (x errIncompletePut) Error() string { return commonMsg } +func (x errIncompletePut) Unwrap() error { + return x.singleErr +} + // WriteObject implements the transformer.ObjectWriter interface. func (t *distributedWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error { t.obj = obj From 64c1392513f43b95bbe3f2afb6bab0a8edd03f89 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 8 Apr 2025 10:45:46 +0300 Subject: [PATCH 35/94] [#1710] object: Sign response even if `CloseAndRecv` returns error * Sign service wraps an error with status and sign a response even if error occurs from `CloseAndRecv` in `Put` and `Patch` methods. Close #1710 Change-Id: I7e1d8fe00db53607fa6e04ebec9a29b87349f8a1 Signed-off-by: Airat Arifullin --- pkg/services/object/sign.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 2b44227a5..fd8e926dd 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -96,7 +96,8 @@ func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.PutRes } else { resp, err = s.stream.CloseAndRecv(ctx) if err != nil { - return nil, fmt.Errorf("could not close stream and receive response: %w", err) + err = fmt.Errorf("could not close stream and receive response: %w", err) + resp = new(object.PutResponse) } } @@ -132,7 +133,8 @@ func (s *patchStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.Patc } else { resp, err = s.stream.CloseAndRecv(ctx) if err != nil { - return nil, fmt.Errorf("could not close stream and receive response: %w", err) + err = fmt.Errorf("could not close stream and receive response: %w", err) + resp = new(object.PatchResponse) } } From e06ecacf57e14630f1cd783af0c88854c26f41cc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 10 Apr 2025 16:04:04 +0300 Subject: [PATCH 36/94] [#1705] engine: Use condition var for evacuation unit tests To know exactly when the evacuation was completed, a conditional variable was added. Closes #1705 Change-Id: I86f6d7d2ad2b9759905b6b5e9341008cb74f5dfd Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/engine.go | 16 +++++++++++----- .../engine/evacuate_limiter.go | 6 +++++- .../engine/evacuate_test.go | 18 +++++++++++++----- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index a915c9bd6..1d33c0592 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -212,12 +212,18 @@ func New(opts ...Option) *StorageEngine { opts[i](c) } + evLimMtx := &sync.RWMutex{} + evLimCond := sync.NewCond(evLimMtx) + return &StorageEngine{ - cfg: c, - shards: make(map[string]hashedShard), - closeCh: make(chan struct{}), - setModeCh: make(chan setModeRequest), - evacuateLimiter: &evacuationLimiter{}, + cfg: c, + shards: make(map[string]hashedShard), + closeCh: make(chan struct{}), + setModeCh: make(chan setModeRequest), + evacuateLimiter: &evacuationLimiter{ + guard: evLimMtx, + statusCond: evLimCond, + }, } } diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index 3dd7494be..b75e8686d 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -139,7 +139,8 @@ type evacuationLimiter struct { eg *errgroup.Group cancel context.CancelFunc - guard sync.RWMutex + guard *sync.RWMutex + statusCond *sync.Cond // used in unit tests } func (l *evacuationLimiter) TryStart(ctx context.Context, shardIDs []string, result *EvacuateShardRes) (*errgroup.Group, context.Context, error) { @@ -165,6 +166,7 @@ func (l *evacuationLimiter) TryStart(ctx context.Context, shardIDs []string, res startedAt: time.Now().UTC(), result: result, } + l.statusCond.Broadcast() return l.eg, egCtx, nil } @@ -180,6 +182,7 @@ func (l *evacuationLimiter) Complete(err error) { l.state.processState = EvacuateProcessStateCompleted l.state.errMessage = errMsq l.state.finishedAt = time.Now().UTC() + l.statusCond.Broadcast() l.eg = nil } @@ -214,6 +217,7 @@ func (l *evacuationLimiter) ResetEvacuationStatus() error { l.state = EvacuationState{} l.eg = nil l.cancel = nil + l.statusCond.Broadcast() return nil } diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 359e49481..f2ba7d994 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -204,11 +204,10 @@ func TestEvacuateShardObjects(t *testing.T) { func testWaitForEvacuationCompleted(t *testing.T, e *StorageEngine) *EvacuationState { var st *EvacuationState var err error - require.Eventually(t, func() bool { - st, err = e.GetEvacuationState(context.Background()) - require.NoError(t, err) - return st.ProcessingStatus() == EvacuateProcessStateCompleted - }, 6*time.Second, 10*time.Millisecond) + e.evacuateLimiter.waitForCompleted() + st, err = e.GetEvacuationState(context.Background()) + require.NoError(t, err) + require.Equal(t, EvacuateProcessStateCompleted, st.ProcessingStatus()) return st } @@ -817,3 +816,12 @@ func TestEvacuateShardObjectsRepOneOnlyBench(t *testing.T) { t.Logf("evacuate took %v\n", time.Since(start)) require.NoError(t, err) } + +func (l *evacuationLimiter) waitForCompleted() { + l.guard.Lock() + defer l.guard.Unlock() + + for l.state.processState != EvacuateProcessStateCompleted { + l.statusCond.Wait() + } +} From dfe2f9956a82c6bee62e443e9e5178d6584ade64 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 24 Mar 2025 15:32:25 +0300 Subject: [PATCH 37/94] [#1619] logger: Filter entries by tags provided in config Change-Id: Ia2a79d6cb2a5eb263fb2e6db3f9cf9f2a7d57118 Signed-off-by: Anton Nikiforov --- cmd/frostfs-ir/config.go | 23 +++++- cmd/frostfs-ir/main.go | 4 ++ cmd/frostfs-node/config.go | 10 ++- cmd/frostfs-node/config/logger/config.go | 16 +++++ cmd/frostfs-node/validate.go | 5 ++ pkg/util/logger/logger.go | 92 +++++++++++++++++++----- pkg/util/logger/tag_string.go | 24 +++++++ pkg/util/logger/tags.go | 75 +++++++++++++++++++ 8 files changed, 231 insertions(+), 18 deletions(-) create mode 100644 pkg/util/logger/tag_string.go create mode 100644 pkg/util/logger/tags.go diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 19b7f05d6..13a747ba6 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -4,12 +4,14 @@ import ( "context" "os" "os/signal" + "strconv" "syscall" configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "github.com/spf13/cast" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -44,11 +46,30 @@ func reloadConfig() error { if err != nil { return err } - log.Reload(logPrm) + err = logPrm.SetTags(loggerTags()) + if err != nil { + return err + } + logger.UpdateLevelForTags(logPrm) return nil } +func loggerTags() [][]string { + var res [][]string + for i := 0; ; i++ { + var item []string + index := strconv.FormatInt(int64(i), 10) + names := cast.ToString(cfg.Get("logger.tags." + index + ".names")) + if names == "" { + break + } + item = append(item, names, cast.ToString(cfg.Get("logger.tags."+index+".level"))) + res = append(res, item) + } + return res +} + func watchForSignal(ctx context.Context, cancel func()) { ch := make(chan os.Signal, 1) signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 114d8e4de..799feb784 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -80,10 +80,14 @@ func main() { exitErr(err) logPrm.SamplingHook = metrics.LogMetrics().GetSamplingHook() logPrm.PrependTimestamp = cfg.GetBool("logger.timestamp") + err = logPrm.SetTags(loggerTags()) + exitErr(err) log, err = logger.NewLogger(logPrm) exitErr(err) + logger.UpdateLevelForTags(logPrm) + ctx, cancel := context.WithCancel(context.Background()) pprofCmp = newPprofComponent() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 431316258..88e45d848 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -109,6 +109,7 @@ type applicationConfiguration struct { destination string timestamp bool options []zap.Option + tags [][]string } ObjectCfg struct { @@ -241,6 +242,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { })} } a.LoggerCfg.options = opts + a.LoggerCfg.tags = loggerconfig.Tags(c) // Object @@ -727,6 +729,7 @@ func initCfg(appCfg *config.Config) *cfg { logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook() log, err := logger.NewLogger(logPrm) fatalOnErr(err) + logger.UpdateLevelForTags(logPrm) c.internals = initInternals(appCfg, log) @@ -1094,6 +1097,11 @@ func (c *cfg) loggerPrm() (logger.Prm, error) { } prm.PrependTimestamp = c.LoggerCfg.timestamp prm.Options = c.LoggerCfg.options + err = prm.SetTags(c.LoggerCfg.tags) + if err != nil { + // not expected since validation should be performed before + return logger.Prm{}, errors.New("incorrect allowed tags format: " + c.LoggerCfg.destination) + } return prm, nil } @@ -1381,7 +1389,7 @@ func (c *cfg) getComponents(ctx context.Context) []dCmp { if err != nil { return err } - c.log.Reload(prm) + logger.UpdateLevelForTags(prm) return nil }}) components = append(components, dCmp{"runtime", func() error { diff --git a/cmd/frostfs-node/config/logger/config.go b/cmd/frostfs-node/config/logger/config.go index ba9eeea2b..20f373184 100644 --- a/cmd/frostfs-node/config/logger/config.go +++ b/cmd/frostfs-node/config/logger/config.go @@ -2,6 +2,7 @@ package loggerconfig import ( "os" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" @@ -60,6 +61,21 @@ func Timestamp(c *config.Config) bool { return config.BoolSafe(c.Sub(subsection), "timestamp") } +// Tags returns the value of "tags" config parameter from "logger" section. +func Tags(c *config.Config) [][]string { + var res [][]string + sub := c.Sub(subsection).Sub("tags") + for i := 0; ; i++ { + s := sub.Sub(strconv.FormatInt(int64(i), 10)) + names := config.StringSafe(s, "names") + if names == "" { + break + } + res = append(res, []string{names, config.StringSafe(s, "level")}) + } + return res +} + // ToLokiConfig extracts loki config. func ToLokiConfig(c *config.Config) loki.Config { hostname, _ := os.Hostname() diff --git a/cmd/frostfs-node/validate.go b/cmd/frostfs-node/validate.go index ae52b9e4a..22d2e0aa9 100644 --- a/cmd/frostfs-node/validate.go +++ b/cmd/frostfs-node/validate.go @@ -30,6 +30,11 @@ func validateConfig(c *config.Config) error { return fmt.Errorf("invalid logger destination: %w", err) } + err = loggerPrm.SetTags(loggerconfig.Tags(c)) + if err != nil { + return fmt.Errorf("invalid list of allowed tags: %w", err) + } + // shard configuration validation shardNum := 0 diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 10c7e8dc9..276847be1 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -13,8 +13,10 @@ import ( // Logger represents a component // for writing messages to log. type Logger struct { - z *zap.Logger - lvl zap.AtomicLevel + z *zap.Logger + c zapcore.Core + t Tag + w bool } // Prm groups Logger's parameters. @@ -39,6 +41,9 @@ type Prm struct { // Options for zap.Logger Options []zap.Option + + // map of tag's bit masks to log level, overrides lvl + tl map[Tag]zapcore.Level } const ( @@ -68,6 +73,12 @@ func (p *Prm) SetDestination(d string) error { return nil } +// SetTags parses list of tags with log level. +func (p *Prm) SetTags(tags [][]string) (err error) { + p.tl, err = parseTags(tags) + return err +} + // NewLogger constructs a new zap logger instance. Constructing with nil // parameters is safe: default values will be used then. // Passing non-nil parameters after a successful creation (non-error) allows @@ -91,10 +102,8 @@ func NewLogger(prm Prm) (*Logger, error) { } func newConsoleLogger(prm Prm) (*Logger, error) { - lvl := zap.NewAtomicLevelAt(prm.level) - c := zap.NewProductionConfig() - c.Level = lvl + c.Level = zap.NewAtomicLevelAt(zap.DebugLevel) c.Encoding = "console" if prm.SamplingHook != nil { c.Sampling.Hook = prm.SamplingHook @@ -115,15 +124,13 @@ func newConsoleLogger(prm Prm) (*Logger, error) { if err != nil { return nil, err } - - l := &Logger{z: lZap, lvl: lvl} + l := &Logger{z: lZap, c: lZap.Core()} + l = l.WithTag(TagMain) return l, nil } func newJournaldLogger(prm Prm) (*Logger, error) { - lvl := zap.NewAtomicLevelAt(prm.level) - c := zap.NewProductionConfig() if prm.SamplingHook != nil { c.Sampling.Hook = prm.SamplingHook @@ -137,7 +144,7 @@ func newJournaldLogger(prm Prm) (*Logger, error) { encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields) - core := zapjournald.NewCore(lvl, encoder, &journald.Journal{}, zapjournald.SyslogFields) + core := zapjournald.NewCore(zap.NewAtomicLevelAt(zap.DebugLevel), encoder, &journald.Journal{}, zapjournald.SyslogFields) coreWithContext := core.With([]zapcore.Field{ zapjournald.SyslogFacility(zapjournald.LogDaemon), zapjournald.SyslogIdentifier(), @@ -161,22 +168,75 @@ func newJournaldLogger(prm Prm) (*Logger, error) { } opts = append(opts, prm.Options...) lZap := zap.New(samplingCore, opts...) - - l := &Logger{z: lZap, lvl: lvl} + l := &Logger{z: lZap, c: lZap.Core()} + l = l.WithTag(TagMain) return l, nil } -func (l *Logger) Reload(prm Prm) { - l.lvl.SetLevel(prm.level) +// With create a child logger with new fields, don't affect the parent. +// Throws panic if tag is unset. +func (l *Logger) With(fields ...zap.Field) *Logger { + if l.t == 0 { + panic("tag is unset") + } + c := *l + c.z = l.z.With(fields...) + // With called under the logger + c.w = true + return &c } -func (l *Logger) With(fields ...zap.Field) *Logger { - return &Logger{z: l.z.With(fields...)} +type core struct { + c zapcore.Core + l zap.AtomicLevel +} + +func (c *core) Enabled(lvl zapcore.Level) bool { + return c.l.Enabled(lvl) +} + +func (c *core) With(fields []zapcore.Field) zapcore.Core { + clone := *c + clone.c = clone.c.With(fields) + return &clone +} + +func (c *core) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + return c.c.Check(e, ce) +} + +func (c *core) Write(e zapcore.Entry, fields []zapcore.Field) error { + return c.c.Write(e, fields) +} + +func (c *core) Sync() error { + return c.c.Sync() +} + +// WithTag is an equivalent of calling [NewLogger] with the same parameters for the current logger. +// Throws panic if provided unsupported tag. +func (l *Logger) WithTag(tag Tag) *Logger { + if tag == 0 || tag > Tag(len(_Tag_index)-1) { + panic("unsupported tag " + tag.String()) + } + if l.w { + panic("unsupported operation for the logger's state") + } + c := *l + c.t = tag + c.z = l.z.WithOptions(zap.WrapCore(func(zapcore.Core) zapcore.Core { + return &core{ + c: l.c.With([]zap.Field{zap.String("tag", tag.String())}), + l: tagToLogLevel[tag], + } + })) + return &c } func NewLoggerWrapper(z *zap.Logger) *Logger { return &Logger{ z: z.WithOptions(zap.AddCallerSkip(1)), + t: TagMain, } } diff --git a/pkg/util/logger/tag_string.go b/pkg/util/logger/tag_string.go new file mode 100644 index 000000000..80cb712eb --- /dev/null +++ b/pkg/util/logger/tag_string.go @@ -0,0 +1,24 @@ +// Code generated by "stringer -type Tag -linecomment"; DO NOT EDIT. + +package logger + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[TagMain-1] +} + +const _Tag_name = "main" + +var _Tag_index = [...]uint8{0, 4} + +func (i Tag) String() string { + i -= 1 + if i >= Tag(len(_Tag_index)-1) { + return "Tag(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _Tag_name[_Tag_index[i]:_Tag_index[i+1]] +} diff --git a/pkg/util/logger/tags.go b/pkg/util/logger/tags.go new file mode 100644 index 000000000..06abcabe5 --- /dev/null +++ b/pkg/util/logger/tags.go @@ -0,0 +1,75 @@ +package logger + +import ( + "fmt" + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +//go:generate stringer -type Tag -linecomment + +type Tag uint8 + +const ( + _ Tag = iota // + TagMain // main + + defaultLevel = zapcore.InfoLevel +) + +var ( + tagToLogLevel = map[Tag]zap.AtomicLevel{} + stringToTag = map[string]Tag{} +) + +func init() { + for i := TagMain; i <= Tag(len(_Tag_index)-1); i++ { + tagToLogLevel[i] = zap.NewAtomicLevelAt(defaultLevel) + stringToTag[i.String()] = i + } +} + +// parseTags returns: +// - map(always instantiated) of tag to custom log level for that tag; +// - error if it occurred(map is empty). +func parseTags(raw [][]string) (map[Tag]zapcore.Level, error) { + m := make(map[Tag]zapcore.Level) + if len(raw) == 0 { + return m, nil + } + for _, item := range raw { + str, level := item[0], item[1] + if len(level) == 0 { + // It is not necessary to parse tags without level, + // because default log level will be used. + continue + } + var l zapcore.Level + err := l.UnmarshalText([]byte(level)) + if err != nil { + return nil, err + } + tmp := strings.Split(str, ",") + for _, tagStr := range tmp { + tag, ok := stringToTag[strings.TrimSpace(tagStr)] + if !ok { + return nil, fmt.Errorf("unsupported tag %s", str) + } + m[tag] = l + } + } + return m, nil +} + +func UpdateLevelForTags(prm Prm) { + for k, v := range tagToLogLevel { + nk, ok := prm.tl[k] + if ok { + v.SetLevel(nk) + } else { + v.SetLevel(prm.level) + } + } +} From 12fc7850dd78d6ea0818f90d90388de8e42d190c Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 29 Jan 2025 14:37:17 +0300 Subject: [PATCH 38/94] [#1619] logger: Set tags for `ir` components Change-Id: Ifab575bc2a3cd83c9001cd68fffaf94c91494043 Signed-off-by: Anton Nikiforov --- config/example/ir.env | 2 ++ config/example/ir.yaml | 3 +++ pkg/innerring/initialization.go | 16 ++++++++-------- pkg/innerring/innerring.go | 2 +- pkg/util/logger/tag_string.go | 8 ++++++-- pkg/util/logger/tags.go | 8 ++++++-- 6 files changed, 26 insertions(+), 13 deletions(-) diff --git a/config/example/ir.env b/config/example/ir.env index ebd91c243..c13044a6e 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -1,5 +1,7 @@ FROSTFS_IR_LOGGER_LEVEL=info FROSTFS_IR_LOGGER_TIMESTAMP=true +FROSTFS_IR_LOGGER_TAGS_0_NAMES="main, morph" +FROSTFS_IR_LOGGER_TAGS_0_LEVEL="debug" FROSTFS_IR_WALLET_PATH=/path/to/wallet.json FROSTFS_IR_WALLET_ADDRESS=NUHtW3eM6a4mmFCgyyr4rj4wygsTKB88XX diff --git a/config/example/ir.yaml b/config/example/ir.yaml index 49f9fd324..a4a006550 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -3,6 +3,9 @@ logger: level: info # Logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" timestamp: true + tags: + - names: "main, morph" # Possible values: `main`, `morph`, `grpc_svc`, `ir`, `processor`. + level: debug wallet: path: /path/to/wallet.json # Path to NEP-6 NEO wallet file diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index f7b71dbe6..3d236641e 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -50,7 +50,7 @@ func (s *Server) initNetmapProcessor(ctx context.Context, cfg *viper.Viper, var err error s.netmapProcessor, err = netmap.New(&netmap.Params{ - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, PoolSize: poolSize, NetmapClient: netmap.NewNetmapClient(s.netmapClient), @@ -159,7 +159,7 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli } else { // create governance processor governanceProcessor, err := governance.New(&governance.Params{ - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, FrostFSClient: frostfsCli, AlphabetState: s, @@ -225,7 +225,7 @@ func (s *Server) initAlphabetProcessor(ctx context.Context, cfg *viper.Viper) er // create alphabet processor s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, PoolSize: poolSize, AlphabetContracts: s.contracts.alphabet, @@ -247,7 +247,7 @@ func (s *Server) initContainerProcessor(ctx context.Context, cfg *viper.Viper, c s.log.Debug(ctx, logs.ContainerContainerWorkerPool, zap.Int("size", poolSize)) // container processor containerProcessor, err := cont.New(&cont.Params{ - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, PoolSize: poolSize, AlphabetState: s, @@ -268,7 +268,7 @@ func (s *Server) initBalanceProcessor(ctx context.Context, cfg *viper.Viper, fro s.log.Debug(ctx, logs.BalanceBalanceWorkerPool, zap.Int("size", poolSize)) // create balance processor balanceProcessor, err := balance.New(&balance.Params{ - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, PoolSize: poolSize, FrostFSClient: frostfsCli, @@ -291,7 +291,7 @@ func (s *Server) initFrostFSMainnetProcessor(ctx context.Context, cfg *viper.Vip s.log.Debug(ctx, logs.FrostFSFrostfsWorkerPool, zap.Int("size", poolSize)) frostfsProcessor, err := frostfs.New(&frostfs.Params{ - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, PoolSize: poolSize, FrostFSContract: s.contracts.frostfs, @@ -342,7 +342,7 @@ func (s *Server) initGRPCServer(ctx context.Context, cfg *viper.Viper, log *logg controlSvc := controlsrv.NewAuditService(controlsrv.New(p, s.netmapClient, s.containerClient, controlsrv.WithAllowedKeys(authKeys), - ), log, audit) + ), log.WithTag(logger.TagGrpcSvc), audit) grpcControlSrv := grpc.NewServer() control.RegisterControlServiceServer(grpcControlSrv, controlSvc) @@ -458,7 +458,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- } morphChain := &chainParams{ - log: s.log, + log: s.log.WithTag(logger.TagMorph), cfg: cfg, key: s.key, name: morphPrefix, diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index ae5661905..3a5137261 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -339,7 +339,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan ) (*Server, error) { var err error server := &Server{ - log: log, + log: log.WithTag(logger.TagIr), irMetrics: metrics, cmode: cmode, } diff --git a/pkg/util/logger/tag_string.go b/pkg/util/logger/tag_string.go index 80cb712eb..a25b3c445 100644 --- a/pkg/util/logger/tag_string.go +++ b/pkg/util/logger/tag_string.go @@ -9,11 +9,15 @@ func _() { // Re-run the stringer command to generate them again. var x [1]struct{} _ = x[TagMain-1] + _ = x[TagMorph-2] + _ = x[TagGrpcSvc-3] + _ = x[TagIr-4] + _ = x[TagProcessor-5] } -const _Tag_name = "main" +const _Tag_name = "mainmorphgrpc_svcirprocessor" -var _Tag_index = [...]uint8{0, 4} +var _Tag_index = [...]uint8{0, 4, 9, 17, 19, 28} func (i Tag) String() string { i -= 1 diff --git a/pkg/util/logger/tags.go b/pkg/util/logger/tags.go index 06abcabe5..51a6f62da 100644 --- a/pkg/util/logger/tags.go +++ b/pkg/util/logger/tags.go @@ -13,8 +13,12 @@ import ( type Tag uint8 const ( - _ Tag = iota // - TagMain // main + _ Tag = iota // + TagMain // main + TagMorph // morph + TagGrpcSvc // grpc_svc + TagIr // ir + TagProcessor // processor defaultLevel = zapcore.InfoLevel ) From 8e87cbee17d3374c55a536bc14f02acb752f9d5f Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 11 Apr 2025 14:44:19 +0300 Subject: [PATCH 39/94] [#1689] ci: Move commit checker out of Jenkinsfile Commit checker is now configured globally for all Gerrit repositories: https://git.frostfs.info/TrueCloudLab/jenkins/pulls/16 This allows us to execute commit-checker independently from the rest of CI suite and re-check commit message format without rerunning other tests. Change-Id: Ib8f899b856482a5dc5d03861171585415ff6b452 Signed-off-by: Vitaliy Potyarkin --- .ci/Jenkinsfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 4ddd36406..4234de160 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -78,10 +78,4 @@ async { } } } - - task('dco') { - container('git.frostfs.info/truecloudlab/commit-check:master') { - sh 'FROM=pull_request_target commit-check' - } - } } From 0d36e93169495b82e0cd23ff0c681c67c020c5e6 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 9 Apr 2025 16:27:03 +0300 Subject: [PATCH 40/94] [#1332] cli/playground: Move command handler selection to separate function Change-Id: I2dcbd85e61960c3cf141b815edab174e308ef858 Signed-off-by: Alexander Chuprov --- .../modules/container/policy_playground.go | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index f747b3252..a9dea13b1 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -260,6 +260,28 @@ Example of usage: }, } +func (repl *policyPlaygroundREPL) handleCommand(args []string) error { + if len(args) == 0 { + return nil + } + + switch args[0] { + case "list", "ls": + return repl.handleLs(args[1:]) + case "add": + return repl.handleAdd(args[1:]) + case "load": + return repl.handleLoad(args[1:]) + case "remove", "rm": + return repl.handleRemove(args[1:]) + case "eval": + return repl.handleEval(args[1:]) + case "help": + return repl.handleHelp(args[1:]) + } + return fmt.Errorf("unknown command %q", args[0]) +} + func (repl *policyPlaygroundREPL) run() error { if len(viper.GetString(commonflags.RPC)) > 0 { key := key.GetOrGenerate(repl.cmd) @@ -277,17 +299,6 @@ func (repl *policyPlaygroundREPL) run() error { } } - cmdHandlers := map[string]func([]string) error{ - "list": repl.handleLs, - "ls": repl.handleLs, - "add": repl.handleAdd, - "load": repl.handleLoad, - "remove": repl.handleRemove, - "rm": repl.handleRemove, - "eval": repl.handleEval, - "help": repl.handleHelp, - } - var cfgCompleter []readline.PrefixCompleterInterface var helpSubItems []readline.PrefixCompleterInterface @@ -326,17 +337,8 @@ func (repl *policyPlaygroundREPL) run() error { } exit = false - parts := strings.Fields(line) - if len(parts) == 0 { - continue - } - cmd := parts[0] - if handler, exists := cmdHandlers[cmd]; exists { - if err := handler(parts[1:]); err != nil { - fmt.Fprintf(repl.console, "error: %v\n", err) - } - } else { - fmt.Fprintf(repl.console, "error: unknown command %q\n", cmd) + if err := repl.handleCommand(strings.Fields(line)); err != nil { + fmt.Fprintf(repl.console, "error: %v\n", err) } } } From 29b4fbe451bb695724e529e757e339f235dd8759 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 9 Apr 2025 16:31:48 +0300 Subject: [PATCH 41/94] [#1332] cli/playground: Add 'netmap-config' flag Change-Id: I4342fb9a6da2a05c18ae4e0ad9f0c71550efc5ef Signed-off-by: Alexander Chuprov --- .../modules/container/policy_playground.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index a9dea13b1..377cf43c6 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -299,6 +299,11 @@ func (repl *policyPlaygroundREPL) run() error { } } + if len(viper.GetString(netmapConfigPath)) > 0 { + err := repl.handleLoad([]string{viper.GetString(netmapConfigPath)}) + commonCmd.ExitOnErr(repl.cmd, "load netmap config error: %w", err) + } + var cfgCompleter []readline.PrefixCompleterInterface var helpSubItems []readline.PrefixCompleterInterface @@ -354,6 +359,14 @@ If a wallet and endpoint is provided, the initial netmap data will be loaded fro }, } +const ( + netmapConfigPath = "netmap-config" + netmapConfigUsage = "Path to the netmap configuration file" +) + func initContainerPolicyPlaygroundCmd() { commonflags.Init(policyPlaygroundCmd) + policyPlaygroundCmd.Flags().String(netmapConfigPath, "", netmapConfigUsage) + + _ = viper.BindPFlag(netmapConfigPath, policyPlaygroundCmd.Flags().Lookup(netmapConfigPath)) } From 3be33b7117900ba1dc715e330545ca6ff5d44dc4 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 9 Apr 2025 16:34:16 +0300 Subject: [PATCH 42/94] [#1706] cli/playground: Mention 'help' in error message for invalid commands Change-Id: Ica1112b907919a6d19fa1bf683f2a952c4c638e4 Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/container/policy_playground.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 377cf43c6..2cc1107ef 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -279,7 +279,7 @@ func (repl *policyPlaygroundREPL) handleCommand(args []string) error { case "help": return repl.handleHelp(args[1:]) } - return fmt.Errorf("unknown command %q", args[0]) + return fmt.Errorf("unknown command %q. See 'help' for assistance", args[0]) } func (repl *policyPlaygroundREPL) run() error { From 5aaa3df533c02763db0dfef0ad5fd63cb0bebc46 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Apr 2025 14:56:07 +0300 Subject: [PATCH 43/94] [#1700] config: Move config struct to qos package Change-Id: Ie642fff5cd1702cda00425628e11f3fd8c514798 Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/config/engine/config_test.go | 36 ++++++---- .../config/engine/shard/limits/config.go | 66 ++++++------------- config/example/node.env | 6 ++ config/example/node.json | 10 +++ config/example/node.yaml | 6 ++ internal/qos/config.go | 31 +++++++++ internal/qos/limiter.go | 15 ++--- internal/qos/validate.go | 12 ++-- 9 files changed, 109 insertions(+), 75 deletions(-) create mode 100644 internal/qos/config.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 88e45d848..298049158 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -385,7 +385,7 @@ func (a *applicationConfiguration) setGCConfig(target *shardCfg, source *shardco } func (a *applicationConfiguration) setLimiter(target *shardCfg, source *shardconfig.Config) error { - limitsConfig := source.Limits() + limitsConfig := source.Limits().ToConfig() limiter, err := qos.NewLimiter(limitsConfig) if err != nil { return err diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 22f26268d..34613ad9e 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -11,10 +11,10 @@ import ( blobovniczaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza" fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" gcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/gc" - limitsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) @@ -135,8 +135,8 @@ func TestEngineSection(t *testing.T) { require.Equal(t, mode.ReadOnly, sc.Mode()) require.Equal(t, 100, sc.RefillMetabaseWorkersCount()) - readLimits := limits.Read() - writeLimits := limits.Write() + readLimits := limits.ToConfig().Read + writeLimits := limits.ToConfig().Write require.Equal(t, 30*time.Second, readLimits.IdleTimeout) require.Equal(t, int64(10_000), readLimits.MaxRunningOps) require.Equal(t, int64(1_000), readLimits.MaxWaitingOps) @@ -144,7 +144,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, int64(1_000), writeLimits.MaxRunningOps) require.Equal(t, int64(100), writeLimits.MaxWaitingOps) require.ElementsMatch(t, readLimits.Tags, - []limitsconfig.IOTagConfig{ + []qos.IOTagConfig{ { Tag: "internal", Weight: toPtr(20), @@ -173,9 +173,14 @@ func TestEngineSection(t *testing.T) { LimitOps: toPtr(25000), Prohibited: true, }, + { + Tag: "treesync", + Weight: toPtr(5), + LimitOps: toPtr(25), + }, }) require.ElementsMatch(t, writeLimits.Tags, - []limitsconfig.IOTagConfig{ + []qos.IOTagConfig{ { Tag: "internal", Weight: toPtr(200), @@ -203,6 +208,11 @@ func TestEngineSection(t *testing.T) { Weight: toPtr(50), LimitOps: toPtr(2500), }, + { + Tag: "treesync", + Weight: toPtr(50), + LimitOps: toPtr(100), + }, }) case 1: require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path()) @@ -259,14 +269,14 @@ func TestEngineSection(t *testing.T) { require.Equal(t, mode.ReadWrite, sc.Mode()) require.Equal(t, shardconfig.RefillMetabaseWorkersCountDefault, sc.RefillMetabaseWorkersCount()) - readLimits := limits.Read() - writeLimits := limits.Write() - require.Equal(t, limitsconfig.DefaultIdleTimeout, readLimits.IdleTimeout) - require.Equal(t, limitsconfig.NoLimit, readLimits.MaxRunningOps) - require.Equal(t, limitsconfig.NoLimit, readLimits.MaxWaitingOps) - require.Equal(t, limitsconfig.DefaultIdleTimeout, writeLimits.IdleTimeout) - require.Equal(t, limitsconfig.NoLimit, writeLimits.MaxRunningOps) - require.Equal(t, limitsconfig.NoLimit, writeLimits.MaxWaitingOps) + readLimits := limits.ToConfig().Read + writeLimits := limits.ToConfig().Write + require.Equal(t, qos.DefaultIdleTimeout, readLimits.IdleTimeout) + require.Equal(t, qos.NoLimit, readLimits.MaxRunningOps) + require.Equal(t, qos.NoLimit, readLimits.MaxWaitingOps) + require.Equal(t, qos.DefaultIdleTimeout, writeLimits.IdleTimeout) + require.Equal(t, qos.NoLimit, writeLimits.MaxRunningOps) + require.Equal(t, qos.NoLimit, writeLimits.MaxWaitingOps) require.Equal(t, 0, len(readLimits.Tags)) require.Equal(t, 0, len(writeLimits.Tags)) } diff --git a/cmd/frostfs-node/config/engine/shard/limits/config.go b/cmd/frostfs-node/config/engine/shard/limits/config.go index 8444d6aa8..ccd1e0000 100644 --- a/cmd/frostfs-node/config/engine/shard/limits/config.go +++ b/cmd/frostfs-node/config/engine/shard/limits/config.go @@ -1,19 +1,13 @@ package limits import ( - "math" "strconv" - "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "github.com/spf13/cast" ) -const ( - NoLimit int64 = math.MaxInt64 - DefaultIdleTimeout = 5 * time.Minute -) - // From wraps config section into Config. func From(c *config.Config) *Config { return (*Config)(c) @@ -23,36 +17,43 @@ func From(c *config.Config) *Config { // which provides access to Shard's limits configurations. type Config config.Config -// Read returns the value of "read" limits config section. -func (x *Config) Read() OpConfig { +func (x *Config) ToConfig() qos.LimiterConfig { + result := qos.LimiterConfig{ + Read: x.read(), + Write: x.write(), + } + panicOnErr(result.Validate()) + return result +} + +func (x *Config) read() qos.OpConfig { return x.parse("read") } -// Write returns the value of "write" limits config section. -func (x *Config) Write() OpConfig { +func (x *Config) write() qos.OpConfig { return x.parse("write") } -func (x *Config) parse(sub string) OpConfig { +func (x *Config) parse(sub string) qos.OpConfig { c := (*config.Config)(x).Sub(sub) - var result OpConfig + var result qos.OpConfig if s := config.Int(c, "max_waiting_ops"); s > 0 { result.MaxWaitingOps = s } else { - result.MaxWaitingOps = NoLimit + result.MaxWaitingOps = qos.NoLimit } if s := config.Int(c, "max_running_ops"); s > 0 { result.MaxRunningOps = s } else { - result.MaxRunningOps = NoLimit + result.MaxRunningOps = qos.NoLimit } if s := config.DurationSafe(c, "idle_timeout"); s > 0 { result.IdleTimeout = s } else { - result.IdleTimeout = DefaultIdleTimeout + result.IdleTimeout = qos.DefaultIdleTimeout } result.Tags = tags(c) @@ -60,43 +61,16 @@ func (x *Config) parse(sub string) OpConfig { return result } -type OpConfig struct { - // MaxWaitingOps returns the value of "max_waiting_ops" config parameter. - // - // Equals NoLimit if the value is not a positive number. - MaxWaitingOps int64 - // MaxRunningOps returns the value of "max_running_ops" config parameter. - // - // Equals NoLimit if the value is not a positive number. - MaxRunningOps int64 - // IdleTimeout returns the value of "idle_timeout" config parameter. - // - // Equals DefaultIdleTimeout if the value is not a valid duration. - IdleTimeout time.Duration - // Tags returns the value of "tags" config parameter. - // - // Equals nil if the value is not a valid tags config slice. - Tags []IOTagConfig -} - -type IOTagConfig struct { - Tag string - Weight *float64 - LimitOps *float64 - ReservedOps *float64 - Prohibited bool -} - -func tags(c *config.Config) []IOTagConfig { +func tags(c *config.Config) []qos.IOTagConfig { c = c.Sub("tags") - var result []IOTagConfig + var result []qos.IOTagConfig for i := 0; ; i++ { tag := config.String(c, strconv.Itoa(i)+".tag") if tag == "" { return result } - var tagConfig IOTagConfig + var tagConfig qos.IOTagConfig tagConfig.Tag = tag v := c.Value(strconv.Itoa(i) + ".weight") diff --git a/config/example/node.env b/config/example/node.env index b7c798ad8..dfb250341 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -181,6 +181,9 @@ FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_TAG=policer FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_WEIGHT=5 FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_LIMIT_OPS=25000 FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_PROHIBITED=true +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_5_TAG=treesync +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_5_WEIGHT=5 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_5_LIMIT_OPS=25 FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_TAG=internal FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_WEIGHT=200 FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_LIMIT_OPS=0 @@ -198,6 +201,9 @@ FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_3_LIMIT_OPS=2500 FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_4_TAG=policer FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_4_WEIGHT=50 FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_4_LIMIT_OPS=2500 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_5_TAG=treesync +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_5_WEIGHT=50 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_5_LIMIT_OPS=100 ## 1 shard ### Flag to refill Metabase from BlobStor diff --git a/config/example/node.json b/config/example/node.json index 2f4413e4d..0b061a3d4 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -254,6 +254,11 @@ "weight": 5, "limit_ops": 25000, "prohibited": true + }, + { + "tag": "treesync", + "weight": 5, + "limit_ops": 25 } ] }, @@ -288,6 +293,11 @@ "tag": "policer", "weight": 50, "limit_ops": 2500 + }, + { + "tag": "treesync", + "weight": 50, + "limit_ops": 100 } ] } diff --git a/config/example/node.yaml b/config/example/node.yaml index 0b6c7b12c..46e4ebdbe 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -253,6 +253,9 @@ storage: weight: 5 limit_ops: 25000 prohibited: true + - tag: treesync + weight: 5 + limit_ops: 25 write: max_running_ops: 1000 max_waiting_ops: 100 @@ -275,6 +278,9 @@ storage: - tag: policer weight: 50 limit_ops: 2500 + - tag: treesync + weight: 50 + limit_ops: 100 1: writecache: diff --git a/internal/qos/config.go b/internal/qos/config.go new file mode 100644 index 000000000..d90b403b5 --- /dev/null +++ b/internal/qos/config.go @@ -0,0 +1,31 @@ +package qos + +import ( + "math" + "time" +) + +const ( + NoLimit int64 = math.MaxInt64 + DefaultIdleTimeout = 5 * time.Minute +) + +type LimiterConfig struct { + Read OpConfig + Write OpConfig +} + +type OpConfig struct { + MaxWaitingOps int64 + MaxRunningOps int64 + IdleTimeout time.Duration + Tags []IOTagConfig +} + +type IOTagConfig struct { + Tag string + Weight *float64 + LimitOps *float64 + ReservedOps *float64 + Prohibited bool +} diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index c73481c2c..5851d7626 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -8,7 +8,6 @@ import ( "sync/atomic" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" "git.frostfs.info/TrueCloudLab/frostfs-qos/scheduling" "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -37,15 +36,15 @@ type scheduler interface { Close() } -func NewLimiter(c *limits.Config) (Limiter, error) { - if err := validateConfig(c); err != nil { +func NewLimiter(c LimiterConfig) (Limiter, error) { + if err := c.Validate(); err != nil { return nil, err } - readScheduler, err := createScheduler(c.Read()) + readScheduler, err := createScheduler(c.Read) if err != nil { return nil, fmt.Errorf("create read scheduler: %w", err) } - writeScheduler, err := createScheduler(c.Write()) + writeScheduler, err := createScheduler(c.Write) if err != nil { return nil, fmt.Errorf("create write scheduler: %w", err) } @@ -63,8 +62,8 @@ func NewLimiter(c *limits.Config) (Limiter, error) { return l, nil } -func createScheduler(config limits.OpConfig) (scheduler, error) { - if len(config.Tags) == 0 && config.MaxWaitingOps == limits.NoLimit { +func createScheduler(config OpConfig) (scheduler, error) { + if len(config.Tags) == 0 && config.MaxWaitingOps == NoLimit { return newSemaphoreScheduler(config.MaxRunningOps), nil } return scheduling.NewMClock( @@ -72,7 +71,7 @@ func createScheduler(config limits.OpConfig) (scheduler, error) { converToSchedulingTags(config.Tags), config.IdleTimeout) } -func converToSchedulingTags(limits []limits.IOTagConfig) map[string]scheduling.TagInfo { +func converToSchedulingTags(limits []IOTagConfig) map[string]scheduling.TagInfo { result := make(map[string]scheduling.TagInfo) for _, tag := range []IOTag{IOTagBackground, IOTagClient, IOTagInternal, IOTagPolicer, IOTagTreeSync, IOTagWritecache} { result[tag.String()] = scheduling.TagInfo{ diff --git a/internal/qos/validate.go b/internal/qos/validate.go index d4475e38b..70f1f24e8 100644 --- a/internal/qos/validate.go +++ b/internal/qos/validate.go @@ -4,8 +4,6 @@ import ( "errors" "fmt" "math" - - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" ) var errWeightsMustBeSpecified = errors.New("invalid weights: weights must be specified for all tags or not specified for any") @@ -14,17 +12,17 @@ type tagConfig struct { Shares, Limit, Reserved *float64 } -func validateConfig(c *limits.Config) error { - if err := validateOpConfig(c.Read()); err != nil { +func (c *LimiterConfig) Validate() error { + if err := validateOpConfig(c.Read); err != nil { return fmt.Errorf("limits 'read' section validation error: %w", err) } - if err := validateOpConfig(c.Write()); err != nil { + if err := validateOpConfig(c.Write); err != nil { return fmt.Errorf("limits 'write' section validation error: %w", err) } return nil } -func validateOpConfig(c limits.OpConfig) error { +func validateOpConfig(c OpConfig) error { if c.MaxRunningOps <= 0 { return fmt.Errorf("invalid 'max_running_ops = %d': must be greater than zero", c.MaxRunningOps) } @@ -40,7 +38,7 @@ func validateOpConfig(c limits.OpConfig) error { return nil } -func validateTags(configTags []limits.IOTagConfig) error { +func validateTags(configTags []IOTagConfig) error { tags := map[IOTag]tagConfig{ IOTagBackground: {}, IOTagClient: {}, From e80632884a2a89c6a5edfcb46cc1a939c9413028 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Apr 2025 16:21:27 +0300 Subject: [PATCH 44/94] [#1700] config: Drop redundant check Target config created on level above, so limiter is always nil. Change-Id: I1896baae5b9ddeed339a7d2b022a9a886589d362 Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 298049158..b167439e0 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -390,9 +390,6 @@ func (a *applicationConfiguration) setLimiter(target *shardCfg, source *shardcon if err != nil { return err } - if target.limiter != nil { - target.limiter.Close() - } target.limiter = limiter return nil } From fd37cea443df4aad4b3a61db0e31fbfa0ae94c33 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Apr 2025 17:08:44 +0300 Subject: [PATCH 45/94] [#1700] engine: Drop unused block execution methods `BlockExecution` and `ResumeExecution` were used only by unit test. So drop them and simplify code. Change-Id: Ib3de324617e8a27fc1f015542ac5e94df5c60a6e Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/control.go | 67 +++---------------- .../engine/control_test.go | 40 ----------- pkg/local_object_storage/engine/engine.go | 5 +- 3 files changed, 10 insertions(+), 102 deletions(-) diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 96b53581e..bf1649f6e 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -22,10 +22,6 @@ type shardInitError struct { // Open opens all StorageEngine's components. func (e *StorageEngine) Open(ctx context.Context) error { - return e.open(ctx) -} - -func (e *StorageEngine) open(ctx context.Context) error { e.mtx.Lock() defer e.mtx.Unlock() @@ -149,11 +145,11 @@ var errClosed = errors.New("storage engine is closed") func (e *StorageEngine) Close(ctx context.Context) error { close(e.closeCh) defer e.wg.Wait() - return e.setBlockExecErr(ctx, errClosed) + return e.closeEngine(ctx) } // closes all shards. Never returns an error, shard errors are logged. -func (e *StorageEngine) close(ctx context.Context) error { +func (e *StorageEngine) closeAllShards(ctx context.Context) error { e.mtx.RLock() defer e.mtx.RUnlock() @@ -176,70 +172,23 @@ func (e *StorageEngine) execIfNotBlocked(op func() error) error { e.blockExec.mtx.RLock() defer e.blockExec.mtx.RUnlock() - if e.blockExec.err != nil { - return e.blockExec.err + if e.blockExec.closed { + return errClosed } return op() } -// sets the flag of blocking execution of all data operations according to err: -// - err != nil, then blocks the execution. If exec wasn't blocked, calls close method -// (if err == errClosed => additionally releases pools and does not allow to resume executions). -// - otherwise, resumes execution. If exec was blocked, calls open method. -// -// Can be called concurrently with exec. In this case it waits for all executions to complete. -func (e *StorageEngine) setBlockExecErr(ctx context.Context, err error) error { +func (e *StorageEngine) closeEngine(ctx context.Context) error { e.blockExec.mtx.Lock() defer e.blockExec.mtx.Unlock() - prevErr := e.blockExec.err - - wasClosed := errors.Is(prevErr, errClosed) - if wasClosed { + if e.blockExec.closed { return errClosed } - e.blockExec.err = err - - if err == nil { - if prevErr != nil { // block -> ok - return e.open(ctx) - } - } else if prevErr == nil { // ok -> block - return e.close(ctx) - } - - // otherwise do nothing - - return nil -} - -// BlockExecution blocks the execution of any data-related operation. All blocked ops will return err. -// To resume the execution, use ResumeExecution method. -// -// Сan be called regardless of the fact of the previous blocking. If execution wasn't blocked, releases all resources -// similar to Close. Can be called concurrently with Close and any data related method (waits for all executions -// to complete). Returns error if any Close has been called before. -// -// Must not be called concurrently with either Open or Init. -// -// Note: technically passing nil error will resume the execution, otherwise, it is recommended to call ResumeExecution -// for this. -func (e *StorageEngine) BlockExecution(err error) error { - return e.setBlockExecErr(context.Background(), err) -} - -// ResumeExecution resumes the execution of any data-related operation. -// To block the execution, use BlockExecution method. -// -// Сan be called regardless of the fact of the previous blocking. If execution was blocked, prepares all resources -// similar to Open. Can be called concurrently with Close and any data related method (waits for all executions -// to complete). Returns error if any Close has been called before. -// -// Must not be called concurrently with either Open or Init. -func (e *StorageEngine) ResumeExecution() error { - return e.setBlockExecErr(context.Background(), nil) + e.blockExec.closed = true + return e.closeAllShards(ctx) } type ReConfiguration struct { diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index a0e658aeb..4ff0ed5ec 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -2,7 +2,6 @@ package engine import ( "context" - "errors" "fmt" "io/fs" "os" @@ -12,17 +11,14 @@ import ( "testing" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" ) @@ -163,42 +159,6 @@ func testEngineFailInitAndReload(t *testing.T, degradedMode bool, opts []shard.O require.Equal(t, 1, shardCount) } -func TestExecBlocks(t *testing.T) { - e := testNewEngine(t).setShardsNum(t, 2).prepare(t).engine // number doesn't matter in this test, 2 is several but not many - - // put some object - obj := testutil.GenerateObjectWithCID(cidtest.ID()) - - addr := object.AddressOf(obj) - - require.NoError(t, Put(context.Background(), e, obj, false)) - - // block executions - errBlock := errors.New("block exec err") - - require.NoError(t, e.BlockExecution(errBlock)) - - // try to exec some op - _, err := Head(context.Background(), e, addr) - require.ErrorIs(t, err, errBlock) - - // resume executions - require.NoError(t, e.ResumeExecution()) - - _, err = Head(context.Background(), e, addr) // can be any data-related op - require.NoError(t, err) - - // close - require.NoError(t, e.Close(context.Background())) - - // try exec after close - _, err = Head(context.Background(), e, addr) - require.Error(t, err) - - // try to resume - require.Error(t, e.ResumeExecution()) -} - func TestPersistentShardID(t *testing.T) { dir := t.TempDir() diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 1d33c0592..376d545d3 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -33,9 +33,8 @@ type StorageEngine struct { wg sync.WaitGroup blockExec struct { - mtx sync.RWMutex - - err error + mtx sync.RWMutex + closed bool } evacuateLimiter *evacuationLimiter } From f37babdc5482334a3c573f09c7b43f43943d7dfa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 14 Apr 2025 09:51:50 +0300 Subject: [PATCH 46/94] [#1700] shard: Lock shard's mode mutex on close To prevent race between GC handlers and close. Change-Id: I06219230964f000f666a56158d3563c760518c3b Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/control.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 72e650c5e..d489b8b0d 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -363,6 +363,7 @@ func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object // Close releases all Shard's components. func (s *Shard) Close(ctx context.Context) error { + unlock := s.lockExclusive() if s.rb != nil { s.rb.Stop(ctx, s.log) } @@ -388,15 +389,19 @@ func (s *Shard) Close(ctx context.Context) error { } } + if s.opsLimiter != nil { + s.opsLimiter.Close() + } + + unlock() + + // GC waits for handlers and remover to complete. Handlers may try to lock shard's lock. + // So to prevent deadlock GC stopping is outside of exclusive lock. // If Init/Open was unsuccessful gc can be nil. if s.gc != nil { s.gc.stop(ctx) } - if s.opsLimiter != nil { - s.opsLimiter.Close() - } - return lastErr } From 48930ec452bc957a1919ee66f91606937cd08261 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 4 Apr 2025 16:59:22 +0300 Subject: [PATCH 47/94] [#1703] cli: Allow reading RPC endpoint from config file Allowed reading an RPC endpoint from a configuration file when getting current epoch in the `object lock` and `bearer create` commands. Close #1703 Change-Id: Iea8509dff2893a02cb63f695d7f532eecd743ed8 Signed-off-by: Aleksey Savchuk --- cmd/frostfs-cli/modules/bearer/create.go | 3 ++- cmd/frostfs-cli/modules/object/lock.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-cli/modules/bearer/create.go b/cmd/frostfs-cli/modules/bearer/create.go index a86506c37..0927788ba 100644 --- a/cmd/frostfs-cli/modules/bearer/create.go +++ b/cmd/frostfs-cli/modules/bearer/create.go @@ -44,6 +44,7 @@ is set to current epoch + n. _ = viper.BindPFlag(commonflags.WalletPath, ff.Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, ff.Lookup(commonflags.Account)) + _ = viper.BindPFlag(commonflags.RPC, ff.Lookup(commonflags.RPC)) }, } @@ -81,7 +82,7 @@ func createToken(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "can't parse --"+notValidBeforeFlag+" flag: %w", err) if iatRelative || expRelative || nvbRelative { - endpoint, _ := cmd.Flags().GetString(commonflags.RPC) + endpoint := viper.GetString(commonflags.RPC) if len(endpoint) == 0 { commonCmd.ExitOnErr(cmd, "can't fetch current epoch: %w", fmt.Errorf("'%s' flag value must be specified", commonflags.RPC)) } diff --git a/cmd/frostfs-cli/modules/object/lock.go b/cmd/frostfs-cli/modules/object/lock.go index 53dd01868..d67db9f0d 100644 --- a/cmd/frostfs-cli/modules/object/lock.go +++ b/cmd/frostfs-cli/modules/object/lock.go @@ -18,6 +18,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/spf13/cobra" + "github.com/spf13/viper" ) // object lock command. @@ -78,7 +79,7 @@ var objectLockCmd = &cobra.Command{ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - endpoint, _ := cmd.Flags().GetString(commonflags.RPC) + endpoint := viper.GetString(commonflags.RPC) currEpoch, err := internalclient.GetCurrentEpoch(ctx, cmd, endpoint) commonCmd.ExitOnErr(cmd, "Request current epoch: %w", err) From 0712c113dee45c81dfacb882f688c18ea39c1ee3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 15 Apr 2025 10:06:05 +0300 Subject: [PATCH 48/94] [#1700] gc: Fix deadlock `HandleExpiredLocks` gets read lock, then `shard.Close` tries to acquire write lock, but `HandleExpiredLocks` calls `inhumeUnlockedIfExpired` or `selectExpired`, that try to acquire read lock again. Change-Id: Ib2ed015e859328045b5a542a4f569e5e0ff8b05b Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 19b6e2d12..a262a52cb 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -391,6 +391,16 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } + s.handleExpiredObjectsUnsafe(ctx, expired) +} + +func (s *Shard) handleExpiredObjectsUnsafe(ctx context.Context, expired []oid.Address) { + select { + case <-ctx.Done(): + return + default: + } + expired, err := s.getExpiredWithLinked(ctx, expired) if err != nil { s.log.Warn(ctx, logs.ShardGCFailedToGetExpiredWithLinked, zap.Error(err)) @@ -611,13 +621,6 @@ func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, onExpiredFo } func (s *Shard) selectExpired(ctx context.Context, epoch uint64, addresses []oid.Address) ([]oid.Address, error) { - s.m.RLock() - defer s.m.RUnlock() - - if s.info.Mode.NoMetabase() { - return nil, ErrDegradedMode - } - release, err := s.opsLimiter.ReadRequest(ctx) if err != nil { return nil, err @@ -728,7 +731,7 @@ func (s *Shard) inhumeUnlockedIfExpired(ctx context.Context, epoch uint64, unloc return } - s.handleExpiredObjects(ctx, expiredUnlocked) + s.handleExpiredObjectsUnsafe(ctx, expiredUnlocked) } // HandleDeletedLocks unlocks all objects which were locked by lockers. From 56d09a99579a59b41f194f4c90a1dd582a9c88bb Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 7 Feb 2025 17:09:08 +0300 Subject: [PATCH 49/94] [#1640] object: Add priority metric based on geo distance Change-Id: I3a7ea4fc4807392bf50e6ff1389c61367c953074 Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config.go | 17 +- cmd/frostfs-node/config/node/config.go | 5 + config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 53 ++++--- .../object_manager/placement/metrics.go | 150 +++++++++++++++++- .../placement/traverser_test.go | 49 ++++++ 8 files changed, 241 insertions(+), 38 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index b167439e0..8ceef2c31 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -247,15 +247,16 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { // Object a.ObjectCfg.tombstoneLifetime = objectconfig.TombstoneLifetime(c) - var pm []placement.Metric - for _, raw := range objectconfig.Get(c).Priority() { - m, err := placement.ParseMetric(raw) - if err != nil { - return err - } - pm = append(pm, m) + locodeDBPath := nodeconfig.LocodeDBPath(c) + parser, err := placement.NewMetricsParser(locodeDBPath) + if err != nil { + return fmt.Errorf("metrics parser creation: %w", err) } - a.ObjectCfg.priorityMetrics = pm + m, err := parser.ParseMetrics(objectconfig.Get(c).Priority()) + if err != nil { + return fmt.Errorf("parse metrics: %w", err) + } + a.ObjectCfg.priorityMetrics = m // Storage Engine diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index 969d77396..18aa254f1 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -217,3 +217,8 @@ func (l PersistentPolicyRulesConfig) NoSync() bool { func CompatibilityMode(c *config.Config) bool { return config.BoolSafe(c.Sub(subsection), "kludge_compatibility_mode") } + +// LocodeDBPath returns path to LOCODE database. +func LocodeDBPath(c *config.Config) string { + return config.String(c.Sub(subsection), "locode_db_path") +} diff --git a/config/example/node.env b/config/example/node.env index dfb250341..b501d3836 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -23,6 +23,7 @@ FROSTFS_NODE_ATTRIBUTE_1="UN-LOCODE:RU MSK" FROSTFS_NODE_RELAY=true FROSTFS_NODE_PERSISTENT_SESSIONS_PATH=/sessions FROSTFS_NODE_PERSISTENT_STATE_PATH=/state +FROSTFS_NODE_LOCODE_DB_PATH=/path/to/locode/db # Tree service section FROSTFS_TREE_ENABLED=true diff --git a/config/example/node.json b/config/example/node.json index 0b061a3d4..b02f43f60 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -37,7 +37,8 @@ }, "persistent_state": { "path": "/state" - } + }, + "locode_db_path": "/path/to/locode/db" }, "grpc": { "0": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 46e4ebdbe..ba32adb82 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -36,6 +36,7 @@ node: path: /sessions # path to persistent session tokens file of Storage node (default: in-memory sessions) persistent_state: path: /state # path to persistent state file of Storage node + "locode_db_path": "/path/to/locode/db" grpc: - endpoint: s01.frostfs.devenv:8080 # endpoint for gRPC server diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 5fe011ece..248b54ea4 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -12,22 +12,23 @@ There are some custom types used for brevity: # Structure -| Section | Description | -|------------------------|---------------------------------------------------------------------| -| `logger` | [Logging parameters](#logger-section) | -| `pprof` | [PProf configuration](#pprof-section) | -| `prometheus` | [Prometheus metrics configuration](#prometheus-section) | -| `control` | [Control service configuration](#control-section) | -| `contracts` | [Override FrostFS contracts hashes](#contracts-section) | -| `morph` | [N3 blockchain client configuration](#morph-section) | -| `apiclient` | [FrostFS API client configuration](#apiclient-section) | -| `policer` | [Policer service configuration](#policer-section) | -| `replicator` | [Replicator service configuration](#replicator-section) | -| `storage` | [Storage engine configuration](#storage-section) | -| `runtime` | [Runtime configuration](#runtime-section) | -| `audit` | [Audit configuration](#audit-section) | -| `multinet` | [Multinet configuration](#multinet-section) | -| `qos` | [QoS configuration](#qos-section) | +| Section | Description | +|--------------|---------------------------------------------------------| +| `node` | [Node parameters](#node-section) | +| `logger` | [Logging parameters](#logger-section) | +| `pprof` | [PProf configuration](#pprof-section) | +| `prometheus` | [Prometheus metrics configuration](#prometheus-section) | +| `control` | [Control service configuration](#control-section) | +| `contracts` | [Override FrostFS contracts hashes](#contracts-section) | +| `morph` | [N3 blockchain client configuration](#morph-section) | +| `apiclient` | [FrostFS API client configuration](#apiclient-section) | +| `policer` | [Policer service configuration](#policer-section) | +| `replicator` | [Replicator service configuration](#replicator-section) | +| `storage` | [Storage engine configuration](#storage-section) | +| `runtime` | [Runtime configuration](#runtime-section) | +| `audit` | [Audit configuration](#audit-section) | +| `multinet` | [Multinet configuration](#multinet-section) | +| `qos` | [QoS configuration](#qos-section) | # `control` section ```yaml @@ -384,17 +385,19 @@ node: path: /sessions persistent_state: path: /state + locode_db_path: "/path/to/locode/db" ``` -| Parameter | Type | Default value | Description | -|-----------------------|---------------------------------------------------------------|---------------|-------------------------------------------------------------------------| -| `key` | `string` | | Path to the binary-encoded private key. | -| `wallet` | [Wallet config](#wallet-subsection) | | Wallet configuration. Has no effect if `key` is provided. | -| `addresses` | `[]string` | | Addresses advertised in the netmap. | -| `attribute` | `[]string` | | Node attributes as a list of key-value pairs in `:` format. | -| `relay` | `bool` | | Enable relay mode. | -| `persistent_sessions` | [Persistent sessions config](#persistent_sessions-subsection) | | Persistent session token store configuration. | -| `persistent_state` | [Persistent state config](#persistent_state-subsection) | | Persistent state configuration. | +| Parameter | Type | Default value | Description | +|-----------------------|---------------------------------------------------------------|---------------|-----------------------------------------------------------------------------------------------------| +| `key` | `string` | | Path to the binary-encoded private key. | +| `wallet` | [Wallet config](#wallet-subsection) | | Wallet configuration. Has no effect if `key` is provided. | +| `addresses` | `[]string` | | Addresses advertised in the netmap. | +| `attribute` | `[]string` | | Node attributes as a list of key-value pairs in `:` format. | +| `relay` | `bool` | | Enable relay mode. | +| `persistent_sessions` | [Persistent sessions config](#persistent_sessions-subsection) | | Persistent session token store configuration. | +| `persistent_state` | [Persistent state config](#persistent_state-subsection) | | Persistent state configuration. | +| `locode_db_path` | `string` | empty | Path to UN/LOCODE [database](https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/) for FrostFS. | ## `wallet` subsection N3 wallet configuration. diff --git a/pkg/services/object_manager/placement/metrics.go b/pkg/services/object_manager/placement/metrics.go index 45e6df339..0f24a9d96 100644 --- a/pkg/services/object_manager/placement/metrics.go +++ b/pkg/services/object_manager/placement/metrics.go @@ -2,24 +2,90 @@ package placement import ( "errors" + "fmt" + "maps" + "math" "strings" + "sync" + "sync/atomic" + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" + locodebolt "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db/boltdb" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) const ( attrPrefix = "$attribute:" + + geoDistance = "$geoDistance" ) type Metric interface { CalculateValue(*netmap.NodeInfo, *netmap.NodeInfo) int } -func ParseMetric(raw string) (Metric, error) { - if attr, found := strings.CutPrefix(raw, attrPrefix); found { - return NewAttributeMetric(attr), nil +type metricsParser struct { + locodeDBPath string + locodes map[string]locodedb.Point +} + +type MetricParser interface { + ParseMetrics([]string) ([]Metric, error) +} + +func NewMetricsParser(locodeDBPath string) (MetricParser, error) { + return &metricsParser{ + locodeDBPath: locodeDBPath, + }, nil +} + +func (p *metricsParser) initLocodes() error { + if len(p.locodes) != 0 { + return nil } - return nil, errors.New("unsupported priority metric") + if len(p.locodeDBPath) > 0 { + p.locodes = make(map[string]locodedb.Point) + locodeDB := locodebolt.New(locodebolt.Prm{ + Path: p.locodeDBPath, + }, + locodebolt.ReadOnly(), + ) + err := locodeDB.Open() + if err != nil { + return err + } + defer locodeDB.Close() + err = locodeDB.IterateOverLocodes(func(k string, v locodedb.Point) { + p.locodes[k] = v + }) + if err != nil { + return err + } + return nil + } + return errors.New("set path to locode database") +} + +func (p *metricsParser) ParseMetrics(priority []string) ([]Metric, error) { + var metrics []Metric + for _, raw := range priority { + if attr, found := strings.CutPrefix(raw, attrPrefix); found { + metrics = append(metrics, NewAttributeMetric(attr)) + } else if raw == geoDistance { + err := p.initLocodes() + if err != nil { + return nil, err + } + if len(p.locodes) == 0 { + return nil, fmt.Errorf("provide locodes database for metric %s", raw) + } + m := NewGeoDistanceMetric(p.locodes) + metrics = append(metrics, m) + } else { + return nil, fmt.Errorf("unsupported priority metric %s", raw) + } + } + return metrics, nil } // attributeMetric describes priority metric based on attribute. @@ -41,3 +107,79 @@ func (am *attributeMetric) CalculateValue(from *netmap.NodeInfo, to *netmap.Node func NewAttributeMetric(attr string) Metric { return &attributeMetric{attribute: attr} } + +// geoDistanceMetric describes priority metric based on attribute. +type geoDistanceMetric struct { + locodes map[string]locodedb.Point + distance *atomic.Pointer[map[string]int] + mtx sync.Mutex +} + +func NewGeoDistanceMetric(locodes map[string]locodedb.Point) Metric { + d := atomic.Pointer[map[string]int]{} + m := make(map[string]int) + d.Store(&m) + gm := &geoDistanceMetric{ + locodes: locodes, + distance: &d, + } + return gm +} + +// CalculateValue return distance in kilometers between current node and provided, +// if coordinates for provided node found. In other case return math.MaxInt. +func (gm *geoDistanceMetric) CalculateValue(from *netmap.NodeInfo, to *netmap.NodeInfo) int { + fl := from.LOCODE() + tl := to.LOCODE() + if fl == tl { + return 0 + } + m := gm.distance.Load() + if v, ok := (*m)[fl+tl]; ok { + return v + } + return gm.calculateDistance(fl, tl) +} + +func (gm *geoDistanceMetric) calculateDistance(from, to string) int { + gm.mtx.Lock() + defer gm.mtx.Unlock() + od := gm.distance.Load() + if v, ok := (*od)[from+to]; ok { + return v + } + nd := maps.Clone(*od) + var dist int + pointFrom, okFrom := gm.locodes[from] + pointTo, okTo := gm.locodes[to] + if okFrom && okTo { + dist = int(distance(pointFrom.Latitude(), pointFrom.Longitude(), pointTo.Latitude(), pointTo.Longitude())) + } else { + dist = math.MaxInt + } + nd[from+to] = dist + gm.distance.Store(&nd) + + return dist +} + +// distance return amount of KM between two points. +// Parameters are latitude and longitude of point 1 and 2 in decimal degrees. +// Original implementation can be found here https://www.geodatasource.com/developers/go. +func distance(lt1 float64, ln1 float64, lt2 float64, ln2 float64) float64 { + radLat1 := math.Pi * lt1 / 180 + radLat2 := math.Pi * lt2 / 180 + radTheta := math.Pi * (ln1 - ln2) / 180 + + dist := math.Sin(radLat1)*math.Sin(radLat2) + math.Cos(radLat1)*math.Cos(radLat2)*math.Cos(radTheta) + + if dist > 1 { + dist = 1 + } + + dist = math.Acos(dist) + dist = dist * 180 / math.Pi + dist = dist * 60 * 1.1515 * 1.609344 + + return dist +} diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index 9c825bf19..d1370f21e 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -601,4 +601,53 @@ func TestTraverserPriorityMetrics(t *testing.T) { next = tr.Next() require.Nil(t, next) }) + + t.Run("one rep one geo metric", func(t *testing.T) { + t.Skip() + selectors := []int{2} + replicas := []int{2} + + nodes, cnr := testPlacement(selectors, replicas) + + // Node_0, PK - ip4/0.0.0.0/tcp/0 + nodes[0][0].SetAttribute("UN-LOCODE", "RU MOW") + // Node_1, PK - ip4/0.0.0.0/tcp/1 + nodes[0][1].SetAttribute("UN-LOCODE", "RU LED") + + sdkNode := testNode(2) + sdkNode.SetAttribute("UN-LOCODE", "FI HEL") + + nodesCopy := copyVectors(nodes) + + parser, err := NewMetricsParser("/path/to/locode_db") + require.NoError(t, err) + m, err := parser.ParseMetrics([]string{geoDistance}) + require.NoError(t, err) + + tr, err := NewTraverser(context.Background(), + ForContainer(cnr), + UseBuilder(&testBuilder{ + vectors: nodesCopy, + }), + WithoutSuccessTracking(), + WithPriorityMetrics(m), + WithNodeState(&nodeState{ + node: &sdkNode, + }), + ) + require.NoError(t, err) + + // Without priority metric `$geoDistance` the order will be: + // [ {Node_0 RU MOW}, {Node_1 RU LED}] + // With priority metric `$geoDistance` the order should be: + // [ {Node_1 RU LED}, {Node_0 RU MOW}] + next := tr.Next() + require.NotNil(t, next) + require.Equal(t, 2, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[1].PublicKey())) + + next = tr.Next() + require.Nil(t, next) + }) } From bf06c4fb4b935410a4ea9b610f4e12f2fe28cb2a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 20:22:05 +0300 Subject: [PATCH 50/94] [#1689] Remove deprecated NodeInfo.IterateNetworkEndpoints() Change-Id: Ic78f18aed11fab34ee3147ceea657296b89fe60c Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/netmap/nodeinfo.go | 4 +- cmd/frostfs-cli/modules/object/nodes.go | 11 ++---- cmd/frostfs-node/cache.go | 22 ++--------- cmd/frostfs-node/netmap.go | 6 ++- cmd/internal/common/netmap.go | 4 +- pkg/core/netmap/nodes.go | 6 ++- pkg/network/validation.go | 6 ++- pkg/services/replicator/pull.go | 7 +--- pkg/services/tree/redirect.go | 41 +++++++++++-------- pkg/services/tree/replicator.go | 46 ++++++++++++---------- pkg/services/tree/sync.go | 12 +++--- 11 files changed, 83 insertions(+), 82 deletions(-) diff --git a/cmd/frostfs-cli/modules/netmap/nodeinfo.go b/cmd/frostfs-cli/modules/netmap/nodeinfo.go index ae4bb329a..316d18d2b 100644 --- a/cmd/frostfs-cli/modules/netmap/nodeinfo.go +++ b/cmd/frostfs-cli/modules/netmap/nodeinfo.go @@ -62,9 +62,9 @@ func prettyPrintNodeInfo(cmd *cobra.Command, i netmap.NodeInfo) { cmd.Println("state:", stateWord) - netmap.IterateNetworkEndpoints(i, func(s string) { + for s := range i.NetworkEndpoints() { cmd.Println("address:", s) - }) + } i.IterateAttributes(func(key, value string) { cmd.Printf("attribute: %s=%s\n", key, value) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 734b557a4..476238651 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sync" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" @@ -460,17 +461,11 @@ func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.N var cli *client.Client var addresses []string if preferInternal, _ := cmd.Flags().GetBool(preferInternalAddressesFlag); preferInternal { - candidate.IterateNetworkEndpoints(func(s string) bool { - addresses = append(addresses, s) - return false - }) + addresses = slices.AppendSeq(addresses, candidate.NetworkEndpoints()) addresses = append(addresses, candidate.ExternalAddresses()...) } else { addresses = append(addresses, candidate.ExternalAddresses()...) - candidate.IterateNetworkEndpoints(func(s string) bool { - addresses = append(addresses, s) - return false - }) + addresses = slices.AppendSeq(addresses, candidate.NetworkEndpoints()) } var lastErr error diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 38cee5837..e5df0a22d 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -235,16 +235,8 @@ func (s *lruNetmapSource) updateCandidates(ctx context.Context, d time.Duration) slices.Compare(n1.ExternalAddresses(), n2.ExternalAddresses()) != 0 { return 1 } - var ne1 []string - n1.IterateNetworkEndpoints(func(s string) bool { - ne1 = append(ne1, s) - return false - }) - var ne2 []string - n2.IterateNetworkEndpoints(func(s string) bool { - ne2 = append(ne2, s) - return false - }) + ne1 := slices.Collect(n1.NetworkEndpoints()) + ne2 := slices.Collect(n2.NetworkEndpoints()) return slices.Compare(ne1, ne2) }) if ret != 0 { @@ -364,15 +356,9 @@ func getNetMapNodesToUpdate(nm *netmapSDK.NetMap, candidates []netmapSDK.NodeInf } nodeEndpoints := make([]string, 0, nm.Nodes()[i].NumberOfNetworkEndpoints()) - nm.Nodes()[i].IterateNetworkEndpoints(func(s string) bool { - nodeEndpoints = append(nodeEndpoints, s) - return false - }) + nodeEndpoints = slices.AppendSeq(nodeEndpoints, nm.Nodes()[i].NetworkEndpoints()) candidateEndpoints := make([]string, 0, cnd.NumberOfNetworkEndpoints()) - cnd.IterateNetworkEndpoints(func(s string) bool { - candidateEndpoints = append(candidateEndpoints, s) - return false - }) + candidateEndpoints = slices.AppendSeq(candidateEndpoints, cnd.NetworkEndpoints()) if slices.Compare(nodeEndpoints, candidateEndpoints) != 0 { update = true tmp.endpoints = candidateEndpoints diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 0e90e7707..6d57edcce 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -124,7 +124,11 @@ func nodeKeyFromNetmap(c *cfg) []byte { func (c *cfg) iterateNetworkAddresses(f func(string) bool) { ni, ok := c.cfgNetmap.state.getNodeInfo() if ok { - ni.IterateNetworkEndpoints(f) + for s := range ni.NetworkEndpoints() { + if f(s) { + return + } + } } } diff --git a/cmd/internal/common/netmap.go b/cmd/internal/common/netmap.go index f550552d2..334b662f8 100644 --- a/cmd/internal/common/netmap.go +++ b/cmd/internal/common/netmap.go @@ -27,9 +27,9 @@ func PrettyPrintNodeInfo(cmd *cobra.Command, node netmap.NodeInfo, cmd.Printf("%sNode %d: %s %s ", indent, index+1, hex.EncodeToString(node.PublicKey()), strState) - netmap.IterateNetworkEndpoints(node, func(endpoint string) { + for endpoint := range node.NetworkEndpoints() { cmd.Printf("%s ", endpoint) - }) + } cmd.Println() if !short { diff --git a/pkg/core/netmap/nodes.go b/pkg/core/netmap/nodes.go index b0c9e1f9e..f01c07b19 100644 --- a/pkg/core/netmap/nodes.go +++ b/pkg/core/netmap/nodes.go @@ -17,7 +17,11 @@ func (x Node) PublicKey() []byte { // IterateAddresses iterates over all announced network addresses // and passes them into f. Handler MUST NOT be nil. func (x Node) IterateAddresses(f func(string) bool) { - (netmap.NodeInfo)(x).IterateNetworkEndpoints(f) + for s := range (netmap.NodeInfo)(x).NetworkEndpoints() { + if f(s) { + return + } + } } // NumberOfAddresses returns number of announced network addresses. diff --git a/pkg/network/validation.go b/pkg/network/validation.go index 92f650119..73a3ef8d7 100644 --- a/pkg/network/validation.go +++ b/pkg/network/validation.go @@ -35,7 +35,11 @@ var ( type NodeEndpointsIterator netmap.NodeInfo func (x NodeEndpointsIterator) IterateAddresses(f func(string) bool) { - (netmap.NodeInfo)(x).IterateNetworkEndpoints(f) + for s := range (netmap.NodeInfo)(x).NetworkEndpoints() { + if f(s) { + return + } + } } func (x NodeEndpointsIterator) NumberOfAddresses() int { diff --git a/pkg/services/replicator/pull.go b/pkg/services/replicator/pull.go index bb38c72ad..216fe4919 100644 --- a/pkg/services/replicator/pull.go +++ b/pkg/services/replicator/pull.go @@ -3,6 +3,7 @@ package replicator import ( "context" "errors" + "slices" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" @@ -42,11 +43,7 @@ func (p *Replicator) HandlePullTask(ctx context.Context, task Task) { if err == nil { break } - var endpoints []string - node.IterateNetworkEndpoints(func(s string) bool { - endpoints = append(endpoints, s) - return false - }) + endpoints := slices.Collect(node.NetworkEndpoints()) p.log.Error(ctx, logs.ReplicatorCouldNotGetObjectFromRemoteStorage, zap.Stringer("object", task.Addr), zap.Error(err), diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index d92c749a8..3dcdc4fc7 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -41,24 +41,15 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo var called bool for _, n := range cntNodes { var stop bool - n.IterateNetworkEndpoints(func(endpoint string) bool { - ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.IterateNetworkEndpoints", - trace.WithAttributes( - attribute.String("endpoint", endpoint), - )) - defer span.End() - - c, err := s.cache.get(ctx, endpoint) - if err != nil { - return false + for endpoint := range n.NetworkEndpoints() { + stop = s.execOnClient(ctx, endpoint, func(c TreeServiceClient) bool { + called = true + return f(c) + }) + if called { + break } - - s.log.Debug(ctx, logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint)) - - called = true - stop = f(c) - return true - }) + } if stop { return nil } @@ -68,3 +59,19 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo } return nil } + +func (s *Service) execOnClient(ctx context.Context, endpoint string, f func(TreeServiceClient) bool) bool { + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.IterateNetworkEndpoints", + trace.WithAttributes( + attribute.String("endpoint", endpoint), + )) + defer span.End() + + c, err := s.cache.get(ctx, endpoint) + if err != nil { + return false + } + + s.log.Debug(ctx, logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint)) + return f(c) +} diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 01a4ffde0..ee40884eb 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -89,29 +89,13 @@ func (s *Service) ReplicateTreeOp(ctx context.Context, n netmapSDK.NodeInfo, req var lastErr error var lastAddr string - n.IterateNetworkEndpoints(func(addr string) bool { - ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationTaskOnEndpoint", - trace.WithAttributes( - attribute.String("public_key", hex.EncodeToString(n.PublicKey())), - attribute.String("address", addr), - ), - ) - defer span.End() - + for addr := range n.NetworkEndpoints() { lastAddr = addr - - c, err := s.cache.get(ctx, addr) - if err != nil { - lastErr = fmt.Errorf("can't create client: %w", err) - return false + lastErr = s.apply(ctx, n, addr, req) + if lastErr == nil { + break } - - ctx, cancel := context.WithTimeout(ctx, s.replicatorTimeout) - _, lastErr = c.Apply(ctx, req) - cancel() - - return lastErr == nil - }) + } if lastErr != nil { if errors.Is(lastErr, errRecentlyFailed) { @@ -130,6 +114,26 @@ func (s *Service) ReplicateTreeOp(ctx context.Context, n netmapSDK.NodeInfo, req return nil } +func (s *Service) apply(ctx context.Context, n netmapSDK.NodeInfo, addr string, req *ApplyRequest) error { + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationTaskOnEndpoint", + trace.WithAttributes( + attribute.String("public_key", hex.EncodeToString(n.PublicKey())), + attribute.String("address", addr), + ), + ) + defer span.End() + + c, err := s.cache.get(ctx, addr) + if err != nil { + return fmt.Errorf("can't create client: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, s.replicatorTimeout) + _, err = c.Apply(ctx, req) + cancel() + return err +} + func (s *Service) replicateLoop(ctx context.Context) { for range s.replicatorWorkerCount { go s.replicationWorker(ctx) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index c3796fbd4..32297f9d7 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -297,27 +297,27 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, for i, n := range nodes { errGroup.Go(func() error { var nodeSynced bool - n.IterateNetworkEndpoints(func(addr string) bool { + for addr := range n.NetworkEndpoints() { var a network.Address if err := a.FromString(addr); err != nil { s.log.Warn(ctx, logs.TreeFailedToParseAddressForTreeSynchronization, zap.Error(err), zap.String("address", addr)) - return false + continue } cc, err := createConnection(a, grpc.WithContextDialer(s.ds.GrpcContextDialer())) if err != nil { s.log.Warn(ctx, logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr)) - return false + continue } - defer cc.Close() err = s.startStream(egCtx, cid, treeID, from, cc, nodeOperationStreams[i]) if err != nil { s.log.Warn(ctx, logs.TreeFailedToRunTreeSynchronizationForSpecificNode, zap.Error(err), zap.String("address", addr)) } nodeSynced = err == nil - return true - }) + _ = cc.Close() + break + } close(nodeOperationStreams[i]) if !nodeSynced { allNodesSynced.Store(false) From e65d578ba93f18c5d4276300fe660f01941f630c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 20:24:46 +0300 Subject: [PATCH 51/94] [#1689] Remove deprecated NodeInfo.IterateAttributes() Change-Id: Ibd07302079efe148903aa6177759232a28616736 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/container/policy_playground.go | 4 ++-- cmd/frostfs-cli/modules/netmap/nodeinfo.go | 4 ++-- cmd/internal/common/netmap.go | 4 ++-- pkg/util/attributes/parser_test.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 2cc1107ef..cf4862b4a 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -40,9 +40,9 @@ func (repl *policyPlaygroundREPL) handleLs(args []string) error { i := 1 for id, node := range repl.nodes { var attrs []string - node.IterateAttributes(func(k, v string) { + for k, v := range node.Attributes() { attrs = append(attrs, fmt.Sprintf("%s:%q", k, v)) - }) + } fmt.Fprintf(repl.console, "\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) i++ } diff --git a/cmd/frostfs-cli/modules/netmap/nodeinfo.go b/cmd/frostfs-cli/modules/netmap/nodeinfo.go index 316d18d2b..5da66dcd9 100644 --- a/cmd/frostfs-cli/modules/netmap/nodeinfo.go +++ b/cmd/frostfs-cli/modules/netmap/nodeinfo.go @@ -66,7 +66,7 @@ func prettyPrintNodeInfo(cmd *cobra.Command, i netmap.NodeInfo) { cmd.Println("address:", s) } - i.IterateAttributes(func(key, value string) { + for key, value := range i.Attributes() { cmd.Printf("attribute: %s=%s\n", key, value) - }) + } } diff --git a/cmd/internal/common/netmap.go b/cmd/internal/common/netmap.go index 334b662f8..5dd1a060e 100644 --- a/cmd/internal/common/netmap.go +++ b/cmd/internal/common/netmap.go @@ -33,9 +33,9 @@ func PrettyPrintNodeInfo(cmd *cobra.Command, node netmap.NodeInfo, cmd.Println() if !short { - node.IterateAttributes(func(key, value string) { + for key, value := range node.Attributes() { cmd.Printf("%s\t%s: %s\n", indent, key, value) - }) + } } } diff --git a/pkg/util/attributes/parser_test.go b/pkg/util/attributes/parser_test.go index 547c8d50b..66581878a 100644 --- a/pkg/util/attributes/parser_test.go +++ b/pkg/util/attributes/parser_test.go @@ -23,12 +23,12 @@ func testAttributeMap(t *testing.T, mSrc, mExp map[string]string) { mExp = mSrc } - node.IterateAttributes(func(key, value string) { + for key, value := range node.Attributes() { v, ok := mExp[key] require.True(t, ok) require.Equal(t, value, v) delete(mExp, key) - }) + } require.Empty(t, mExp) } From 2d1232ce6de2f9291528c1c3e325bb8a0683de10 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 20:31:35 +0300 Subject: [PATCH 52/94] [#1689] network,core/netmap: Replace Iterate*() functions with iterators Change-Id: I4842a3160d74c56d99ea9465d4be2f0662080605 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config/node/config.go | 10 ++++------ pkg/core/client/util.go | 3 ++- pkg/core/netmap/nodes.go | 12 +++++++++++- pkg/network/group.go | 18 +++++++++--------- pkg/network/group_test.go | 8 ++++---- pkg/network/validation.go | 9 +++------ 6 files changed, 33 insertions(+), 27 deletions(-) diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index 18aa254f1..81b191e96 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -3,7 +3,9 @@ package nodeconfig import ( "fmt" "io/fs" + "iter" "os" + "slices" "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" @@ -88,12 +90,8 @@ func Wallet(c *config.Config) *keys.PrivateKey { type stringAddressGroup []string -func (x stringAddressGroup) IterateAddresses(f func(string) bool) { - for i := range x { - if f(x[i]) { - break - } - } +func (x stringAddressGroup) Addresses() iter.Seq[string] { + return slices.Values(x) } func (x stringAddressGroup) NumberOfAddresses() int { diff --git a/pkg/core/client/util.go b/pkg/core/client/util.go index d4bc0cf68..91ee5c6c3 100644 --- a/pkg/core/client/util.go +++ b/pkg/core/client/util.go @@ -3,6 +3,7 @@ package client import ( "bytes" "fmt" + "iter" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -19,7 +20,7 @@ func nodeInfoFromKeyAddr(dst *NodeInfo, k []byte, a, external network.AddressGro // Args must not be nil. func NodeInfoFromRawNetmapElement(dst *NodeInfo, info interface { PublicKey() []byte - IterateAddresses(func(string) bool) + Addresses() iter.Seq[string] NumberOfAddresses() int ExternalAddresses() []string }, diff --git a/pkg/core/netmap/nodes.go b/pkg/core/netmap/nodes.go index f01c07b19..e58e42634 100644 --- a/pkg/core/netmap/nodes.go +++ b/pkg/core/netmap/nodes.go @@ -1,6 +1,10 @@ package netmap -import "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" +import ( + "iter" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" +) // Node is a named type of netmap.NodeInfo which provides interface needed // in the current repository. Node is expected to be used everywhere instead @@ -14,8 +18,14 @@ func (x Node) PublicKey() []byte { return (netmap.NodeInfo)(x).PublicKey() } +// Addresses returns an iterator over all announced network addresses. +func (x Node) Addresses() iter.Seq[string] { + return (netmap.NodeInfo)(x).NetworkEndpoints() +} + // IterateAddresses iterates over all announced network addresses // and passes them into f. Handler MUST NOT be nil. +// Deprecated: use [Node.Addresses] instead. func (x Node) IterateAddresses(f func(string) bool) { for s := range (netmap.NodeInfo)(x).NetworkEndpoints() { if f(s) { diff --git a/pkg/network/group.go b/pkg/network/group.go index 5a71e530e..0044fb2d4 100644 --- a/pkg/network/group.go +++ b/pkg/network/group.go @@ -3,6 +3,7 @@ package network import ( "errors" "fmt" + "iter" "slices" "sort" @@ -68,9 +69,8 @@ func (x AddressGroup) Swap(i, j int) { // MultiAddressIterator is an interface of network address group. type MultiAddressIterator interface { - // IterateAddresses must iterate over network addresses and pass each one - // to the handler until it returns true. - IterateAddresses(func(string) bool) + // Addresses must return an iterator over network addresses. + Addresses() iter.Seq[string] // NumberOfAddresses must return number of addresses in group. NumberOfAddresses() int @@ -131,19 +131,19 @@ func (x *AddressGroup) FromIterator(iter MultiAddressIterator) error { // iterateParsedAddresses parses each address from MultiAddressIterator and passes it to f // until 1st parsing failure or f's error. func iterateParsedAddresses(iter MultiAddressIterator, f func(s Address) error) (err error) { - iter.IterateAddresses(func(s string) bool { + for s := range iter.Addresses() { var a Address err = a.FromString(s) if err != nil { - err = fmt.Errorf("could not parse address from string: %w", err) - return true + return fmt.Errorf("could not parse address from string: %w", err) } err = f(a) - - return err != nil - }) + if err != nil { + return err + } + } return } diff --git a/pkg/network/group_test.go b/pkg/network/group_test.go index 5b335fa52..d08264533 100644 --- a/pkg/network/group_test.go +++ b/pkg/network/group_test.go @@ -1,6 +1,8 @@ package network import ( + "iter" + "slices" "sort" "testing" @@ -58,10 +60,8 @@ func TestAddressGroup_FromIterator(t *testing.T) { type testIterator []string -func (t testIterator) IterateAddresses(f func(string) bool) { - for i := range t { - f(t[i]) - } +func (t testIterator) Addresses() iter.Seq[string] { + return slices.Values(t) } func (t testIterator) NumberOfAddresses() int { diff --git a/pkg/network/validation.go b/pkg/network/validation.go index 73a3ef8d7..b5157f28f 100644 --- a/pkg/network/validation.go +++ b/pkg/network/validation.go @@ -2,6 +2,7 @@ package network import ( "errors" + "iter" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) @@ -34,12 +35,8 @@ var ( // MultiAddressIterator. type NodeEndpointsIterator netmap.NodeInfo -func (x NodeEndpointsIterator) IterateAddresses(f func(string) bool) { - for s := range (netmap.NodeInfo)(x).NetworkEndpoints() { - if f(s) { - return - } - } +func (x NodeEndpointsIterator) Addresses() iter.Seq[string] { + return (netmap.NodeInfo)(x).NetworkEndpoints() } func (x NodeEndpointsIterator) NumberOfAddresses() int { From 98308d0cad312c34f2575d05651405de58c5b2f3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 14 Apr 2025 15:39:56 +0300 Subject: [PATCH 53/94] [#1715] blobstor: Allow to specify custom compression level Change-Id: I140c39b9dceaaeb58767061b131777af22242b19 Signed-off-by: Dmitrii Stepanov --- .../maintenance/zombie/storage_engine.go | 2 ++ cmd/frostfs-node/config.go | 4 +++ cmd/frostfs-node/config/engine/config_test.go | 2 ++ .../config/engine/shard/config.go | 10 ++++++ config/example/node.env | 1 + config/example/node.json | 1 + config/example/node.yaml | 5 +-- docs/storage-node-configuration.md | 3 +- internal/assert/cond.go | 4 +++ internal/logs/logs.go | 1 + pkg/local_object_storage/blobstor/blobstor.go | 6 ++++ .../blobstor/compression/compress.go | 34 ++++++++++++++++++- pkg/local_object_storage/blobstor/control.go | 5 +++ 13 files changed, 74 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go index 5851e049c..fe75a6f6f 100644 --- a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -135,6 +136,7 @@ func getMetabaseOpts(sh *shardconfig.Config) []meta.Option { func getBlobstorOpts(ctx context.Context, sh *shardconfig.Config) []blobstor.Option { result := []blobstor.Option{ blobstor.WithCompressObjects(sh.Compress()), + blobstor.WithCompressionLevel(compression.Level(sh.CompressionLevel())), blobstor.WithUncompressableContentTypes(sh.UncompressableContentTypes()), blobstor.WithCompressibilityEstimate(sh.EstimateCompressibility()), blobstor.WithCompressibilityEstimateThreshold(sh.EstimateCompressibilityThreshold()), diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 8ceef2c31..f32953c58 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -40,6 +40,7 @@ import ( netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -129,6 +130,7 @@ type applicationConfiguration struct { type shardCfg struct { compress bool + compressionLevel compression.Level estimateCompressibility bool estimateCompressibilityThreshold float64 @@ -273,6 +275,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, source *s target.refillMetabaseWorkersCount = source.RefillMetabaseWorkersCount() target.mode = source.Mode() target.compress = source.Compress() + target.compressionLevel = compression.Level(source.CompressionLevel()) target.estimateCompressibility = source.EstimateCompressibility() target.estimateCompressibilityThreshold = source.EstimateCompressibilityThreshold() target.uncompressableContentType = source.UncompressableContentTypes() @@ -1027,6 +1030,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID blobstoreOpts := []blobstor.Option{ blobstor.WithCompressObjects(shCfg.compress), + blobstor.WithCompressionLevel(shCfg.compressionLevel), blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), blobstor.WithCompressibilityEstimate(shCfg.estimateCompressibility), blobstor.WithCompressibilityEstimateThreshold(shCfg.estimateCompressibilityThreshold), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 34613ad9e..afadb4c99 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -101,6 +101,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 10*time.Millisecond, meta.BoltDB().MaxBatchDelay()) require.Equal(t, true, sc.Compress()) + require.Equal(t, "fastest", sc.CompressionLevel()) require.Equal(t, []string{"audio/*", "video/*"}, sc.UncompressableContentTypes()) require.Equal(t, true, sc.EstimateCompressibility()) require.Equal(t, float64(0.7), sc.EstimateCompressibilityThreshold()) @@ -237,6 +238,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 20*time.Millisecond, meta.BoltDB().MaxBatchDelay()) require.Equal(t, false, sc.Compress()) + require.Equal(t, "", sc.CompressionLevel()) require.Equal(t, []string(nil), sc.UncompressableContentTypes()) require.EqualValues(t, 102400, sc.SmallSizeLimit()) diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go index e50d56b95..14e91f01f 100644 --- a/cmd/frostfs-node/config/engine/shard/config.go +++ b/cmd/frostfs-node/config/engine/shard/config.go @@ -37,6 +37,16 @@ func (x *Config) Compress() bool { ) } +// CompressionLevel returns the value of "compression_level" config parameter. +// +// Returns empty string if the value is not a valid string. +func (x *Config) CompressionLevel() string { + return config.StringSafe( + (*config.Config)(x), + "compression_level", + ) +} + // UncompressableContentTypes returns the value of "compress_skip_content_types" config parameter. // // Returns nil if a the value is missing or is invalid. diff --git a/config/example/node.env b/config/example/node.env index b501d3836..fb4fc69dd 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -123,6 +123,7 @@ FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_SIZE=100 FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_DELAY=10ms ### Blobstor config FROSTFS_STORAGE_SHARD_0_COMPRESS=true +FROSTFS_STORAGE_SHARD_0_COMPRESSION_LEVEL=fastest FROSTFS_STORAGE_SHARD_0_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*" FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY=true FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY_THRESHOLD=0.7 diff --git a/config/example/node.json b/config/example/node.json index b02f43f60..1e9dd7835 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -184,6 +184,7 @@ "max_batch_delay": "10ms" }, "compress": true, + "compression_level": "fastest", "compression_exclude_content_types": [ "audio/*", "video/*" ], diff --git a/config/example/node.yaml b/config/example/node.yaml index ba32adb82..26c6e248c 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -160,7 +160,7 @@ storage: max_batch_delay: 5ms # maximum delay for a batch of operations to be executed max_batch_size: 100 # maximum amount of operations in a single batch - compress: false # turn on/off zstd(level 3) compression of stored objects + compress: false # turn on/off zstd compression of stored objects small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes blobstor: @@ -202,7 +202,8 @@ storage: max_batch_size: 100 max_batch_delay: 10ms - compress: true # turn on/off zstd(level 3) compression of stored objects + compress: true # turn on/off zstd compression of stored objects + compression_level: fastest compression_exclude_content_types: - audio/* - video/* diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 248b54ea4..19f6ee66d 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -188,6 +188,7 @@ The following table describes configuration for each shard. | Parameter | Type | Default value | Description | | ------------------------------------------------ | ------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `compress` | `bool` | `false` | Flag to enable compression. | +| `compression_level` | `string` | `optimal` | Compression level. Available values are `optimal`, `fastest`, `smallest_size`. | | `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | | `compression_estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. | | `compression_estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. | @@ -199,7 +200,7 @@ The following table describes configuration for each shard. | `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | | `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. | | `gc` | [GC config](#gc-subsection) | | GC configuration. | -| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. | +| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. | ### `blobstor` subsection diff --git a/internal/assert/cond.go b/internal/assert/cond.go index c6a034f94..113d2eba9 100644 --- a/internal/assert/cond.go +++ b/internal/assert/cond.go @@ -23,3 +23,7 @@ func NoError(err error, details ...string) { panic(content) } } + +func Fail(details ...string) { + panic(strings.Join(details, " ")) +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 3a3ceb150..dedc7e12c 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -516,4 +516,5 @@ const ( FailedToValidateIncomingIOTag = "failed to validate incoming IO tag, replaced with `client`" WriteCacheFailedToAcquireRPSQuota = "writecache failed to acquire RPS quota to flush object" FailedToUpdateNetmapCandidates = "update netmap candidates failed" + UnknownCompressionLevelDefaultWillBeUsed = "unknown compression level, 'optimal' will be used" ) diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index d7c333349..7a9568aff 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -109,6 +109,12 @@ func WithCompressObjects(comp bool) Option { } } +func WithCompressionLevel(level compression.Level) Option { + return func(c *cfg) { + c.compression.Level = level + } +} + // WithCompressibilityEstimate returns an option to use // normilized compressibility estimate to decide compress // data or not. diff --git a/pkg/local_object_storage/blobstor/compression/compress.go b/pkg/local_object_storage/blobstor/compression/compress.go index 85ab47692..8a86b6982 100644 --- a/pkg/local_object_storage/blobstor/compression/compress.go +++ b/pkg/local_object_storage/blobstor/compression/compress.go @@ -4,15 +4,26 @@ import ( "bytes" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/klauspost/compress" "github.com/klauspost/compress/zstd" ) +type Level string + +const ( + LevelDefault Level = "" + LevelOptimal Level = "optimal" + LevelFastest Level = "fastest" + LevelSmallestSize Level = "smallest_size" +) + // Config represents common compression-related configuration. type Config struct { Enabled bool UncompressableContentTypes []string + Level Level UseCompressEstimation bool CompressEstimationThreshold float64 @@ -30,7 +41,7 @@ func (c *Config) Init() error { var err error if c.Enabled { - c.encoder, err = zstd.NewWriter(nil) + c.encoder, err = zstd.NewWriter(nil, zstd.WithEncoderLevel(c.compressionLevel())) if err != nil { return err } @@ -116,3 +127,24 @@ func (c *Config) Close() error { } return err } + +func (c *Config) HasValidCompressionLevel() bool { + return c.Level == LevelDefault || + c.Level == LevelOptimal || + c.Level == LevelFastest || + c.Level == LevelSmallestSize +} + +func (c *Config) compressionLevel() zstd.EncoderLevel { + switch c.Level { + case LevelDefault, LevelOptimal: + return zstd.SpeedDefault + case LevelFastest: + return zstd.SpeedFastest + case LevelSmallestSize: + return zstd.SpeedBestCompression + default: + assert.Fail("unknown compression level", string(c.Level)) + return zstd.SpeedDefault + } +} diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 93316be02..0418eedd0 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -6,6 +6,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "go.uber.org/zap" ) @@ -53,6 +54,10 @@ var ErrInitBlobovniczas = errors.New("failure on blobovnicza initialization stag func (b *BlobStor) Init(ctx context.Context) error { b.log.Debug(ctx, logs.BlobstorInitializing) + if !b.compression.HasValidCompressionLevel() { + b.log.Warn(ctx, logs.UnknownCompressionLevelDefaultWillBeUsed, zap.String("level", string(b.compression.Level))) + b.compression.Level = compression.LevelDefault + } if err := b.compression.Init(); err != nil { return err } From 8c746a914ac917a16b0ab3098e70fda7e15dcbb0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 14 Apr 2025 16:18:08 +0300 Subject: [PATCH 54/94] [#1715] compression: Decouple Config and Compressor Refactoring. Change-Id: Ide2e1378f30c39045d4bacd13a902331bd4f764f Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/blobovnicza.go | 4 ++-- .../blobstor/blobovniczatree/option.go | 2 +- pkg/local_object_storage/blobstor/blobstor.go | 4 ++-- .../blobstor/common/storage.go | 4 ++-- .../blobstor/compression/bench_test.go | 16 +++++++++----- .../blobstor/compression/compress.go | 22 +++++++++++-------- .../blobstor/fstree/fstree.go | 18 +++++++-------- .../blobstor/memstore/control.go | 4 ++-- .../blobstor/memstore/option.go | 2 +- .../blobstor/teststore/option.go | 8 +++---- .../blobstor/teststore/teststore.go | 4 ++-- .../writecache/writecache.go | 2 +- 12 files changed, 49 insertions(+), 41 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index d9e99d0d1..3e8b9f07b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -158,11 +158,11 @@ func (b *Blobovniczas) Path() string { } // SetCompressor implements common.Storage. -func (b *Blobovniczas) SetCompressor(cc *compression.Config) { +func (b *Blobovniczas) SetCompressor(cc *compression.Compressor) { b.compression = cc } -func (b *Blobovniczas) Compressor() *compression.Config { +func (b *Blobovniczas) Compressor() *compression.Compressor { return b.compression } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 0e1b2022e..2f6d31b4e 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -19,7 +19,7 @@ type cfg struct { openedCacheSize int blzShallowDepth uint64 blzShallowWidth uint64 - compression *compression.Config + compression *compression.Compressor blzOpts []blobovnicza.Option reportError func(context.Context, string, error) // reportError is the function called when encountering disk errors. metrics Metrics diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index 7a9568aff..cf67c6fe9 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -41,7 +41,7 @@ type SubStorageInfo struct { type Option func(*cfg) type cfg struct { - compression compression.Config + compression compression.Compressor log *logger.Logger storage []SubStorage metrics Metrics @@ -158,6 +158,6 @@ func WithMetrics(m Metrics) Option { } } -func (b *BlobStor) Compressor() *compression.Config { +func (b *BlobStor) Compressor() *compression.Compressor { return &b.compression } diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 6ecef48cd..e35c35e60 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -18,8 +18,8 @@ type Storage interface { Path() string ObjectsCount(ctx context.Context) (uint64, error) - SetCompressor(cc *compression.Config) - Compressor() *compression.Config + SetCompressor(cc *compression.Compressor) + Compressor() *compression.Compressor // SetReportErrorFunc allows to provide a function to be called on disk errors. // This function MUST be called before Open. diff --git a/pkg/local_object_storage/blobstor/compression/bench_test.go b/pkg/local_object_storage/blobstor/compression/bench_test.go index 9f70f8ec2..445a0494b 100644 --- a/pkg/local_object_storage/blobstor/compression/bench_test.go +++ b/pkg/local_object_storage/blobstor/compression/bench_test.go @@ -11,7 +11,7 @@ import ( ) func BenchmarkCompression(b *testing.B) { - c := Config{Enabled: true} + c := Compressor{Config: Config{Enabled: true}} require.NoError(b, c.Init()) for _, size := range []int{128, 1024, 32 * 1024, 32 * 1024 * 1024} { @@ -33,7 +33,7 @@ func BenchmarkCompression(b *testing.B) { } } -func benchWith(b *testing.B, c Config, data []byte) { +func benchWith(b *testing.B, c Compressor, data []byte) { b.ResetTimer() b.ReportAllocs() for range b.N { @@ -56,8 +56,10 @@ func BenchmarkCompressionRealVSEstimate(b *testing.B) { b.Run("estimate", func(b *testing.B) { b.ResetTimer() - c := &Config{ - Enabled: true, + c := &Compressor{ + Config: Config{ + Enabled: true, + }, } require.NoError(b, c.Init()) @@ -76,8 +78,10 @@ func BenchmarkCompressionRealVSEstimate(b *testing.B) { b.Run("compress", func(b *testing.B) { b.ResetTimer() - c := &Config{ - Enabled: true, + c := &Compressor{ + Config: Config{ + Enabled: true, + }, } require.NoError(b, c.Init()) diff --git a/pkg/local_object_storage/blobstor/compression/compress.go b/pkg/local_object_storage/blobstor/compression/compress.go index 8a86b6982..efe84ea2a 100644 --- a/pkg/local_object_storage/blobstor/compression/compress.go +++ b/pkg/local_object_storage/blobstor/compression/compress.go @@ -19,6 +19,13 @@ const ( LevelSmallestSize Level = "smallest_size" ) +type Compressor struct { + Config + + encoder *zstd.Encoder + decoder *zstd.Decoder +} + // Config represents common compression-related configuration. type Config struct { Enabled bool @@ -27,9 +34,6 @@ type Config struct { UseCompressEstimation bool CompressEstimationThreshold float64 - - encoder *zstd.Encoder - decoder *zstd.Decoder } // zstdFrameMagic contains first 4 bytes of any compressed object @@ -37,7 +41,7 @@ type Config struct { var zstdFrameMagic = []byte{0x28, 0xb5, 0x2f, 0xfd} // Init initializes compression routines. -func (c *Config) Init() error { +func (c *Compressor) Init() error { var err error if c.Enabled { @@ -84,7 +88,7 @@ func (c *Config) NeedsCompression(obj *objectSDK.Object) bool { // Decompress decompresses data if it starts with the magic // and returns data untouched otherwise. -func (c *Config) Decompress(data []byte) ([]byte, error) { +func (c *Compressor) Decompress(data []byte) ([]byte, error) { if len(data) < 4 || !bytes.Equal(data[:4], zstdFrameMagic) { return data, nil } @@ -93,7 +97,7 @@ func (c *Config) Decompress(data []byte) ([]byte, error) { // Compress compresses data if compression is enabled // and returns data untouched otherwise. -func (c *Config) Compress(data []byte) []byte { +func (c *Compressor) Compress(data []byte) []byte { if c == nil || !c.Enabled { return data } @@ -107,7 +111,7 @@ func (c *Config) Compress(data []byte) []byte { return c.compress(data) } -func (c *Config) compress(data []byte) []byte { +func (c *Compressor) compress(data []byte) []byte { maxSize := c.encoder.MaxEncodedSize(len(data)) compressed := c.encoder.EncodeAll(data, make([]byte, 0, maxSize)) if len(data) < len(compressed) { @@ -117,7 +121,7 @@ func (c *Config) compress(data []byte) []byte { } // Close closes encoder and decoder, returns any error occurred. -func (c *Config) Close() error { +func (c *Compressor) Close() error { var err error if c.encoder != nil { err = c.encoder.Close() @@ -135,7 +139,7 @@ func (c *Config) HasValidCompressionLevel() bool { c.Level == LevelSmallestSize } -func (c *Config) compressionLevel() zstd.EncoderLevel { +func (c *Compressor) compressionLevel() zstd.EncoderLevel { switch c.Level { case LevelDefault, LevelOptimal: return zstd.SpeedDefault diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 031b385b2..112741ab4 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -45,7 +45,7 @@ type FSTree struct { log *logger.Logger - *compression.Config + compressor *compression.Compressor Depth uint64 DirNameLen int @@ -82,7 +82,7 @@ func New(opts ...Option) *FSTree { Permissions: 0o700, RootPath: "./", }, - Config: nil, + compressor: nil, Depth: 4, DirNameLen: DirNameLen, metrics: &noopMetrics{}, @@ -196,7 +196,7 @@ func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, pr } if err == nil { - data, err = t.Decompress(data) + data, err = t.compressor.Decompress(data) } if err != nil { if prm.IgnoreErrors { @@ -405,7 +405,7 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err return common.PutRes{}, err } if !prm.DontCompress { - prm.RawData = t.Compress(prm.RawData) + prm.RawData = t.compressor.Compress(prm.RawData) } size = len(prm.RawData) @@ -448,7 +448,7 @@ func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, err } } - data, err = t.Decompress(data) + data, err = t.compressor.Decompress(data) if err != nil { return common.GetRes{}, err } @@ -597,12 +597,12 @@ func (t *FSTree) Path() string { } // SetCompressor implements common.Storage. -func (t *FSTree) SetCompressor(cc *compression.Config) { - t.Config = cc +func (t *FSTree) SetCompressor(cc *compression.Compressor) { + t.compressor = cc } -func (t *FSTree) Compressor() *compression.Config { - return t.Config +func (t *FSTree) Compressor() *compression.Compressor { + return t.compressor } // SetReportErrorFunc implements common.Storage. diff --git a/pkg/local_object_storage/blobstor/memstore/control.go b/pkg/local_object_storage/blobstor/memstore/control.go index 95a916662..3df96a1c3 100644 --- a/pkg/local_object_storage/blobstor/memstore/control.go +++ b/pkg/local_object_storage/blobstor/memstore/control.go @@ -16,7 +16,7 @@ func (s *memstoreImpl) Init() error func (s *memstoreImpl) Close(context.Context) error { return nil } func (s *memstoreImpl) Type() string { return Type } func (s *memstoreImpl) Path() string { return s.rootPath } -func (s *memstoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc } -func (s *memstoreImpl) Compressor() *compression.Config { return s.compression } +func (s *memstoreImpl) SetCompressor(cc *compression.Compressor) { s.compression = cc } +func (s *memstoreImpl) Compressor() *compression.Compressor { return s.compression } func (s *memstoreImpl) SetReportErrorFunc(func(context.Context, string, error)) {} func (s *memstoreImpl) SetParentID(string) {} diff --git a/pkg/local_object_storage/blobstor/memstore/option.go b/pkg/local_object_storage/blobstor/memstore/option.go index 97a03993d..7605af4e5 100644 --- a/pkg/local_object_storage/blobstor/memstore/option.go +++ b/pkg/local_object_storage/blobstor/memstore/option.go @@ -7,7 +7,7 @@ import ( type cfg struct { rootPath string readOnly bool - compression *compression.Config + compression *compression.Compressor } func defaultConfig() *cfg { diff --git a/pkg/local_object_storage/blobstor/teststore/option.go b/pkg/local_object_storage/blobstor/teststore/option.go index fb1188751..3a38ecf82 100644 --- a/pkg/local_object_storage/blobstor/teststore/option.go +++ b/pkg/local_object_storage/blobstor/teststore/option.go @@ -17,8 +17,8 @@ type cfg struct { Type func() string Path func() string - SetCompressor func(cc *compression.Config) - Compressor func() *compression.Config + SetCompressor func(cc *compression.Compressor) + Compressor func() *compression.Compressor SetReportErrorFunc func(f func(context.Context, string, error)) Get func(common.GetPrm) (common.GetRes, error) @@ -45,11 +45,11 @@ func WithClose(f func() error) Option { return func(c *cfg) { c func WithType(f func() string) Option { return func(c *cfg) { c.overrides.Type = f } } func WithPath(f func() string) Option { return func(c *cfg) { c.overrides.Path = f } } -func WithSetCompressor(f func(*compression.Config)) Option { +func WithSetCompressor(f func(*compression.Compressor)) Option { return func(c *cfg) { c.overrides.SetCompressor = f } } -func WithCompressor(f func() *compression.Config) Option { +func WithCompressor(f func() *compression.Compressor) Option { return func(c *cfg) { c.overrides.Compressor = f } } diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 626ba0023..190b6a876 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -116,7 +116,7 @@ func (s *TestStore) Path() string { } } -func (s *TestStore) SetCompressor(cc *compression.Config) { +func (s *TestStore) SetCompressor(cc *compression.Compressor) { s.mu.RLock() defer s.mu.RUnlock() switch { @@ -129,7 +129,7 @@ func (s *TestStore) SetCompressor(cc *compression.Config) { } } -func (s *TestStore) Compressor() *compression.Config { +func (s *TestStore) Compressor() *compression.Compressor { s.mu.RLock() defer s.mu.RUnlock() switch { diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 70b17eb8e..7ed511318 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -52,7 +52,7 @@ type Cache interface { // MainStorage is the interface of the underlying storage of Cache implementations. type MainStorage interface { - Compressor() *compression.Config + Compressor() *compression.Compressor Exists(context.Context, common.ExistsPrm) (common.ExistsRes, error) Put(context.Context, common.PutPrm) (common.PutRes, error) } From 0ee7467da5c9cda8c398a011fa00c86657b7cbc3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 14 Apr 2025 17:36:51 +0300 Subject: [PATCH 55/94] [#1715] config: Add `compression` config section To group all `compression_*` parameters together. Change-Id: I11ad9600f731903753fef1adfbc0328ef75bbf87 Signed-off-by: Dmitrii Stepanov --- .../maintenance/zombie/storage_engine.go | 7 +-- cmd/frostfs-node/config.go | 18 +----- cmd/frostfs-node/config/engine/config_test.go | 17 +++--- .../config/engine/shard/config.go | 56 ++++++------------- config/example/node.env | 2 +- config/example/node.json | 20 ++++--- config/example/node.yaml | 18 +++--- docs/storage-node-configuration.md | 51 +++++++++++------ pkg/local_object_storage/blobstor/blobstor.go | 47 +--------------- .../blobstor/blobstor_test.go | 11 +++- .../blobstor/compression/compress.go | 8 +-- .../blobstor/iterate_test.go | 5 +- 12 files changed, 105 insertions(+), 155 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go index fe75a6f6f..c54b331f3 100644 --- a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -135,11 +134,7 @@ func getMetabaseOpts(sh *shardconfig.Config) []meta.Option { func getBlobstorOpts(ctx context.Context, sh *shardconfig.Config) []blobstor.Option { result := []blobstor.Option{ - blobstor.WithCompressObjects(sh.Compress()), - blobstor.WithCompressionLevel(compression.Level(sh.CompressionLevel())), - blobstor.WithUncompressableContentTypes(sh.UncompressableContentTypes()), - blobstor.WithCompressibilityEstimate(sh.EstimateCompressibility()), - blobstor.WithCompressibilityEstimateThreshold(sh.EstimateCompressibilityThreshold()), + blobstor.WithCompression(sh.Compression()), blobstor.WithStorages(getSubStorages(ctx, sh)), blobstor.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index f32953c58..f80401b5b 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -129,13 +129,9 @@ type applicationConfiguration struct { } type shardCfg struct { - compress bool - compressionLevel compression.Level - estimateCompressibility bool - estimateCompressibilityThreshold float64 + compression compression.Config smallSizeObjectLimit uint64 - uncompressableContentType []string refillMetabase bool refillMetabaseWorkersCount int mode shardmode.Mode @@ -274,11 +270,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, source *s target.refillMetabase = source.RefillMetabase() target.refillMetabaseWorkersCount = source.RefillMetabaseWorkersCount() target.mode = source.Mode() - target.compress = source.Compress() - target.compressionLevel = compression.Level(source.CompressionLevel()) - target.estimateCompressibility = source.EstimateCompressibility() - target.estimateCompressibilityThreshold = source.EstimateCompressibilityThreshold() - target.uncompressableContentType = source.UncompressableContentTypes() + target.compression = source.Compression() target.smallSizeObjectLimit = source.SmallSizeLimit() a.setShardWriteCacheConfig(&target, source) @@ -1029,11 +1021,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID ss := c.getSubstorageOpts(ctx, shCfg) blobstoreOpts := []blobstor.Option{ - blobstor.WithCompressObjects(shCfg.compress), - blobstor.WithCompressionLevel(shCfg.compressionLevel), - blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), - blobstor.WithCompressibilityEstimate(shCfg.estimateCompressibility), - blobstor.WithCompressibilityEstimateThreshold(shCfg.estimateCompressibilityThreshold), + blobstor.WithCompression(shCfg.compression), blobstor.WithStorages(ss), blobstor.WithLogger(c.log), } diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index afadb4c99..401c54edc 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -15,6 +15,7 @@ import ( writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) @@ -100,11 +101,11 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 100, meta.BoltDB().MaxBatchSize()) require.Equal(t, 10*time.Millisecond, meta.BoltDB().MaxBatchDelay()) - require.Equal(t, true, sc.Compress()) - require.Equal(t, "fastest", sc.CompressionLevel()) - require.Equal(t, []string{"audio/*", "video/*"}, sc.UncompressableContentTypes()) - require.Equal(t, true, sc.EstimateCompressibility()) - require.Equal(t, float64(0.7), sc.EstimateCompressibilityThreshold()) + require.Equal(t, true, sc.Compression().Enabled) + require.Equal(t, compression.LevelFastest, sc.Compression().Level) + require.Equal(t, []string{"audio/*", "video/*"}, sc.Compression().UncompressableContentTypes) + require.Equal(t, true, sc.Compression().EstimateCompressibility) + require.Equal(t, float64(0.7), sc.Compression().EstimateCompressibilityThreshold) require.EqualValues(t, 102400, sc.SmallSizeLimit()) require.Equal(t, 2, len(ss)) @@ -237,9 +238,9 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 200, meta.BoltDB().MaxBatchSize()) require.Equal(t, 20*time.Millisecond, meta.BoltDB().MaxBatchDelay()) - require.Equal(t, false, sc.Compress()) - require.Equal(t, "", sc.CompressionLevel()) - require.Equal(t, []string(nil), sc.UncompressableContentTypes()) + require.Equal(t, false, sc.Compression().Enabled) + require.Equal(t, compression.LevelDefault, sc.Compression().Level) + require.Equal(t, []string(nil), sc.Compression().UncompressableContentTypes) require.EqualValues(t, 102400, sc.SmallSizeLimit()) require.Equal(t, 2, len(ss)) diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go index 14e91f01f..d42646da7 100644 --- a/cmd/frostfs-node/config/engine/shard/config.go +++ b/cmd/frostfs-node/config/engine/shard/config.go @@ -8,6 +8,7 @@ import ( metabaseconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/metabase" piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) @@ -27,52 +28,27 @@ func From(c *config.Config) *Config { return (*Config)(c) } -// Compress returns the value of "compress" config parameter. -// -// Returns false if the value is not a valid bool. -func (x *Config) Compress() bool { - return config.BoolSafe( - (*config.Config)(x), - "compress", - ) -} - -// CompressionLevel returns the value of "compression_level" config parameter. -// -// Returns empty string if the value is not a valid string. -func (x *Config) CompressionLevel() string { - return config.StringSafe( - (*config.Config)(x), - "compression_level", - ) -} - -// UncompressableContentTypes returns the value of "compress_skip_content_types" config parameter. -// -// Returns nil if a the value is missing or is invalid. -func (x *Config) UncompressableContentTypes() []string { - return config.StringSliceSafe( - (*config.Config)(x), - "compression_exclude_content_types") -} - -// EstimateCompressibility returns the value of "estimate_compressibility" config parameter. -// -// Returns false if the value is not a valid bool. -func (x *Config) EstimateCompressibility() bool { - return config.BoolSafe( - (*config.Config)(x), - "compression_estimate_compressibility", - ) +func (x *Config) Compression() compression.Config { + cc := (*config.Config)(x).Sub("compression") + if cc == nil { + return compression.Config{} + } + return compression.Config{ + Enabled: config.BoolSafe(cc, "enabled"), + UncompressableContentTypes: config.StringSliceSafe(cc, "exclude_content_types"), + Level: compression.Level(config.StringSafe(cc, "level")), + EstimateCompressibility: config.BoolSafe(cc, "estimate_compressibility"), + EstimateCompressibilityThreshold: estimateCompressibilityThreshold(cc), + } } // EstimateCompressibilityThreshold returns the value of "estimate_compressibility_threshold" config parameter. // // Returns EstimateCompressibilityThresholdDefault if the value is not defined, not valid float or not in range [0.0; 1.0]. -func (x *Config) EstimateCompressibilityThreshold() float64 { +func estimateCompressibilityThreshold(c *config.Config) float64 { v := config.FloatOrDefault( - (*config.Config)(x), - "compression_estimate_compressibility_threshold", + c, + "estimate_compressibility_threshold", EstimateCompressibilityThresholdDefault) if v < 0.0 || v > 1.0 { return EstimateCompressibilityThresholdDefault diff --git a/config/example/node.env b/config/example/node.env index fb4fc69dd..9d054fe78 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -122,7 +122,7 @@ FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644 FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_SIZE=100 FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_DELAY=10ms ### Blobstor config -FROSTFS_STORAGE_SHARD_0_COMPRESS=true +FROSTFS_STORAGE_SHARD_0_COMPRESSION_ENABLED=true FROSTFS_STORAGE_SHARD_0_COMPRESSION_LEVEL=fastest FROSTFS_STORAGE_SHARD_0_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*" FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY=true diff --git a/config/example/node.json b/config/example/node.json index 1e9dd7835..110e99ee8 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -183,13 +183,15 @@ "max_batch_size": 100, "max_batch_delay": "10ms" }, - "compress": true, - "compression_level": "fastest", - "compression_exclude_content_types": [ - "audio/*", "video/*" - ], - "compression_estimate_compressibility": true, - "compression_estimate_compressibility_threshold": 0.7, + "compression": { + "enabled": true, + "level": "fastest", + "exclude_content_types": [ + "audio/*", "video/*" + ], + "estimate_compressibility": true, + "estimate_compressibility_threshold": 0.7 + }, "small_object_size": 102400, "blobstor": [ { @@ -323,7 +325,9 @@ "max_batch_size": 200, "max_batch_delay": "20ms" }, - "compress": false, + "compression": { + "enabled": false + }, "small_object_size": 102400, "blobstor": [ { diff --git a/config/example/node.yaml b/config/example/node.yaml index 26c6e248c..de5eaa133 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -160,7 +160,8 @@ storage: max_batch_delay: 5ms # maximum delay for a batch of operations to be executed max_batch_size: 100 # maximum amount of operations in a single batch - compress: false # turn on/off zstd compression of stored objects + compression: + enabled: false # turn on/off zstd compression of stored objects small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes blobstor: @@ -202,13 +203,14 @@ storage: max_batch_size: 100 max_batch_delay: 10ms - compress: true # turn on/off zstd compression of stored objects - compression_level: fastest - compression_exclude_content_types: - - audio/* - - video/* - compression_estimate_compressibility: true - compression_estimate_compressibility_threshold: 0.7 + compression: + enabled: true # turn on/off zstd compression of stored objects + level: fastest + exclude_content_types: + - audio/* + - video/* + estimate_compressibility: true + estimate_compressibility_threshold: 0.7 blobstor: - type: blobovnicza diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 19f6ee66d..1494d2fca 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -185,22 +185,41 @@ Contains configuration for each shard. Keys must be consecutive numbers starting `default` subsection has the same format and specifies defaults for missing values. The following table describes configuration for each shard. -| Parameter | Type | Default value | Description | -| ------------------------------------------------ | ------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `compress` | `bool` | `false` | Flag to enable compression. | -| `compression_level` | `string` | `optimal` | Compression level. Available values are `optimal`, `fastest`, `smallest_size`. | -| `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | -| `compression_estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. | -| `compression_estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. | -| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | -| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | -| `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. | -| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | -| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | -| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | -| `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. | -| `gc` | [GC config](#gc-subsection) | | GC configuration. | -| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. | +| Parameter | Type | Default value | Description | +| ------------------------------ | --------------------------------------------- | ------------- | --------------------------------------------------------------------------------------------------------- | +| `compression` | [Compression config](#compression-subsection) | | Compression config. | +| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | +| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | +| `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. | +| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | +| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | +| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | +| `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. | +| `gc` | [GC config](#gc-subsection) | | GC configuration. | +| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. | + +### `compression` subsection + +Contains compression config. + +```yaml +compression: + enabled: true + level: smallest_size + exclude_content_types: + - audio/* + - video/* + estimate_compressibility: true + estimate_compressibility_threshold: 0.7 +``` + +| Parameter | Type | Default value | Description | +| ------------------------------------ | ---------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | `bool` | `false` | Flag to enable compression. | +| `level` | `string` | `optimal` | Compression level. Available values are `optimal`, `fastest`, `smallest_size`. | +| `exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | +| `estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. | +| `estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. | ### `blobstor` subsection diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index cf67c6fe9..edb2c1946 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -95,52 +95,9 @@ func WithLogger(l *logger.Logger) Option { } } -// WithCompressObjects returns option to toggle -// compression of the stored objects. -// -// If true, Zstandard algorithm is used for data compression. -// -// If compressor (decompressor) creation failed, -// the uncompressed option will be used, and the error -// is recorded in the provided log. -func WithCompressObjects(comp bool) Option { +func WithCompression(comp compression.Config) Option { return func(c *cfg) { - c.compression.Enabled = comp - } -} - -func WithCompressionLevel(level compression.Level) Option { - return func(c *cfg) { - c.compression.Level = level - } -} - -// WithCompressibilityEstimate returns an option to use -// normilized compressibility estimate to decide compress -// data or not. -// -// See https://github.com/klauspost/compress/blob/v1.17.2/compressible.go#L5 -func WithCompressibilityEstimate(v bool) Option { - return func(c *cfg) { - c.compression.UseCompressEstimation = v - } -} - -// WithCompressibilityEstimateThreshold returns an option to set -// normilized compressibility estimate threshold. -// -// See https://github.com/klauspost/compress/blob/v1.17.2/compressible.go#L5 -func WithCompressibilityEstimateThreshold(threshold float64) Option { - return func(c *cfg) { - c.compression.CompressEstimationThreshold = threshold - } -} - -// WithUncompressableContentTypes returns option to disable decompression -// for specific content types as seen by object.AttributeContentType attribute. -func WithUncompressableContentTypes(values []string) Option { - return func(c *cfg) { - c.compression.UncompressableContentTypes = values + c.compression.Config = comp } } diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index 6cc56fa3b..6ddeb6f00 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -51,7 +52,9 @@ func TestCompression(t *testing.T) { newBlobStor := func(t *testing.T, compress bool) *BlobStor { bs := New( - WithCompressObjects(compress), + WithCompression(compression.Config{ + Enabled: compress, + }), WithStorages(defaultStorages(dir, smallSizeLimit))) require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bs.Init(context.Background())) @@ -113,8 +116,10 @@ func TestBlobstor_needsCompression(t *testing.T) { dir := t.TempDir() bs := New( - WithCompressObjects(compress), - WithUncompressableContentTypes(ct), + WithCompression(compression.Config{ + Enabled: compress, + UncompressableContentTypes: ct, + }), WithStorages([]SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( diff --git a/pkg/local_object_storage/blobstor/compression/compress.go b/pkg/local_object_storage/blobstor/compression/compress.go index efe84ea2a..c76cec9a1 100644 --- a/pkg/local_object_storage/blobstor/compression/compress.go +++ b/pkg/local_object_storage/blobstor/compression/compress.go @@ -32,8 +32,8 @@ type Config struct { UncompressableContentTypes []string Level Level - UseCompressEstimation bool - CompressEstimationThreshold float64 + EstimateCompressibility bool + EstimateCompressibilityThreshold float64 } // zstdFrameMagic contains first 4 bytes of any compressed object @@ -101,9 +101,9 @@ func (c *Compressor) Compress(data []byte) []byte { if c == nil || !c.Enabled { return data } - if c.UseCompressEstimation { + if c.EstimateCompressibility { estimated := compress.Estimate(data) - if estimated >= c.CompressEstimationThreshold { + if estimated >= c.EstimateCompressibilityThreshold { return c.compress(data) } return data diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index ccfa510fe..2786321a8 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -8,6 +8,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -24,7 +25,9 @@ func TestIterateObjects(t *testing.T) { // create BlobStor instance blobStor := New( WithStorages(defaultStorages(p, smalSz)), - WithCompressObjects(true), + WithCompression(compression.Config{ + Enabled: true, + }), ) defer os.RemoveAll(p) From 410b6f70bae0e44da29d328ec3ba532042617c8b Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Tue, 15 Apr 2025 17:56:47 +0300 Subject: [PATCH 56/94] [#1716] cli: Return trace ID on operation failure Close #1716 Change-Id: I293d0cc6b7331517e8cde42eae07d65384976da5 Signed-off-by: Aleksey Savchuk --- cmd/internal/common/exit.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/internal/common/exit.go b/cmd/internal/common/exit.go index b8acf0143..13f447af4 100644 --- a/cmd/internal/common/exit.go +++ b/cmd/internal/common/exit.go @@ -51,8 +51,13 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) { } cmd.PrintErrln(err) - if cmd.PersistentPostRun != nil { - cmd.PersistentPostRun(cmd, nil) + for p := cmd; p != nil; p = p.Parent() { + if p.PersistentPostRun != nil { + p.PersistentPostRun(cmd, nil) + if !cobra.EnableTraverseRunHooks { + break + } + } } os.Exit(code) } From a285d8924f87da6af50ad3fc28b5ac5726fc92e8 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 14 Apr 2025 12:38:52 +0300 Subject: [PATCH 57/94] [#1693] node: Replace conditional panics with asserts Change-Id: I5024705fd1693d00cb9241235030a73984c2a7e1 Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-node/netmap.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 6d57edcce..82b799e4c 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -8,6 +8,7 @@ import ( "net" "sync/atomic" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -104,9 +105,7 @@ func (s *networkState) getNodeInfo() (res netmapSDK.NodeInfo, ok bool) { v := s.nodeInfo.Load() if v != nil { res, ok = v.(netmapSDK.NodeInfo) - if !ok { - panic(fmt.Sprintf("unexpected value in atomic node info state: %T", v)) - } + assert.True(ok, fmt.Sprintf("unexpected value in atomic node info state: %T", v)) } return From fc6abe30b892491fce0053492220e2ec91e11236 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 8 Apr 2025 16:47:52 +0300 Subject: [PATCH 58/94] [#1693] storage: Replace conditional panics with asserts Change-Id: I9d8ccde3c71fca716856c7bfc53da20ee0542f20 Signed-off-by: Ekaterina Lebedeva --- .../blobstor/fstree/counter.go | 17 +++++++---------- pkg/local_object_storage/metabase/lock.go | 5 ++--- pkg/local_object_storage/metabase/util.go | 5 ++--- pkg/local_object_storage/writecache/limiter.go | 16 ++++++---------- 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/counter.go b/pkg/local_object_storage/blobstor/fstree/counter.go index b5dbc9e40..3caee7ee1 100644 --- a/pkg/local_object_storage/blobstor/fstree/counter.go +++ b/pkg/local_object_storage/blobstor/fstree/counter.go @@ -2,6 +2,8 @@ package fstree import ( "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" ) // FileCounter used to count files in FSTree. The implementation must be thread-safe. @@ -52,16 +54,11 @@ func (c *SimpleCounter) Dec(size uint64) { c.mtx.Lock() defer c.mtx.Unlock() - if c.count > 0 { - c.count-- - } else { - panic("fstree.SimpleCounter: invalid count") - } - if c.size >= size { - c.size -= size - } else { - panic("fstree.SimpleCounter: invalid size") - } + assert.True(c.count > 0, "fstree.SimpleCounter: invalid count") + c.count-- + + assert.True(c.size >= size, "fstree.SimpleCounter: invalid size") + c.size -= size } func (c *SimpleCounter) CountSize() (uint64, uint64) { diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index f73c2b4f6..f4cb9e53b 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -7,6 +7,7 @@ import ( "slices" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -63,9 +64,7 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. return ErrReadOnlyMode } - if len(locked) == 0 { - panic("empty locked list") - } + assert.False(len(locked) == 0, "empty locked list") err := db.lockInternal(locked, cnr, locker) success = err == nil diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index 80851f1c4..4ad83332b 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -278,9 +279,7 @@ func objectKey(obj oid.ID, key []byte) []byte { // // firstIrregularObjectType(tx, cnr, obj) usage allows getting object type. func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) objectSDK.Type { - if len(objs) == 0 { - panic("empty object list in firstIrregularObjectType") - } + assert.False(len(objs) == 0, "empty object list in firstIrregularObjectType") var keys [2][1 + cidSize]byte diff --git a/pkg/local_object_storage/writecache/limiter.go b/pkg/local_object_storage/writecache/limiter.go index ddc4101be..0e020b36e 100644 --- a/pkg/local_object_storage/writecache/limiter.go +++ b/pkg/local_object_storage/writecache/limiter.go @@ -3,6 +3,8 @@ package writecache import ( "errors" "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" ) var errLimiterClosed = errors.New("acquire failed: limiter closed") @@ -45,17 +47,11 @@ func (l *flushLimiter) release(size uint64) { l.cond.L.Lock() defer l.cond.L.Unlock() - if l.size >= size { - l.size -= size - } else { - panic("flushLimiter: invalid size") - } + assert.True(l.size >= size, "flushLimiter: invalid size") + l.size -= size - if l.count > 0 { - l.count-- - } else { - panic("flushLimiter: invalid count") - } + assert.True(l.count > 0, "flushLimiter: invalid count") + l.count-- l.cond.Broadcast() } From bc045b29e21b458c9d3f244f05a16bce9869059e Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 8 Apr 2025 16:54:05 +0300 Subject: [PATCH 59/94] [#1693] services: Replace conditional panics with asserts Change-Id: Ic79609e6ad867caa88ad245b3014aa7fc32e05a8 Signed-off-by: Ekaterina Lebedeva --- pkg/services/netmap/executor.go | 11 +++++++---- pkg/services/object_manager/tombstone/constructor.go | 5 ++--- .../object_manager/tombstone/source/source.go | 5 ++--- pkg/services/policer/policer.go | 6 +++--- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index 44101a153..1b92fdaad 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/version" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" @@ -46,10 +47,12 @@ type NetworkInfo interface { } func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo, respSvc *response.Service) Server { - if s == nil || netInfo == nil || !version.IsValid(v) || respSvc == nil { - // this should never happen, otherwise it programmers bug - panic("can't create netmap execution service") - } + // this should never happen, otherwise it's a programmer's bug + msg := "BUG: can't create netmap execution service" + assert.False(s == nil, msg, "node state is nil") + assert.False(netInfo == nil, msg, "network info is nil") + assert.False(respSvc == nil, msg, "response service is nil") + assert.True(version.IsValid(v), msg, "invalid version") res := &executorSvc{ state: s, diff --git a/pkg/services/object_manager/tombstone/constructor.go b/pkg/services/object_manager/tombstone/constructor.go index 67ddf316f..2147a32fe 100644 --- a/pkg/services/object_manager/tombstone/constructor.go +++ b/pkg/services/object_manager/tombstone/constructor.go @@ -3,6 +3,7 @@ package tombstone import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "go.uber.org/zap" @@ -49,9 +50,7 @@ func NewChecker(oo ...Option) *ExpirationChecker { panicOnNil(cfg.tsSource, "Tombstone source") cache, err := lru.New[string, uint64](cfg.cacheSize) - if err != nil { - panic(fmt.Errorf("could not create LRU cache with %d size: %w", cfg.cacheSize, err)) - } + assert.NoError(err, fmt.Sprintf("could not create LRU cache with %d size", cfg.cacheSize)) return &ExpirationChecker{ cache: cache, diff --git a/pkg/services/object_manager/tombstone/source/source.go b/pkg/services/object_manager/tombstone/source/source.go index 1ff07b05a..975941847 100644 --- a/pkg/services/object_manager/tombstone/source/source.go +++ b/pkg/services/object_manager/tombstone/source/source.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -38,9 +39,7 @@ func (s *TombstoneSourcePrm) SetGetService(v *getsvc.Service) { // Panics if any of the provided options does not allow // constructing a valid tombstone local Source. func NewSource(p TombstoneSourcePrm) Source { - if p.s == nil { - panic("Tombstone source: nil object service") - } + assert.False(p.s == nil, "Tombstone source: nil object service") return Source(p) } diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index 4e8bacfec..d18b71a23 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -1,9 +1,11 @@ package policer import ( + "fmt" "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" "go.uber.org/zap" @@ -57,9 +59,7 @@ func New(opts ...Option) *Policer { c.log = c.log.With(zap.String("component", "Object Policer")) cache, err := lru.New[oid.Address, time.Time](int(c.cacheSize)) - if err != nil { - panic(err) - } + assert.NoError(err, fmt.Sprintf("could not create LRU cache with %d size", c.cacheSize)) return &Policer{ cfg: c, From 5dd8d7e87ab20ff3d7e1c4c2d93982a1ae79381a Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 8 Apr 2025 16:54:51 +0300 Subject: [PATCH 60/94] [#1693] network: Replace conditional panics with asserts Change-Id: Icba39aa2ed0048d63c6efed398273627e1e4fbbe Signed-off-by: Ekaterina Lebedeva --- pkg/network/address.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/network/address.go b/pkg/network/address.go index cb83a813d..4643eef15 100644 --- a/pkg/network/address.go +++ b/pkg/network/address.go @@ -2,11 +2,11 @@ package network import ( "errors" - "fmt" "net" "net/url" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" @@ -44,11 +44,9 @@ func (a Address) equal(addr Address) bool { // See also FromString. func (a Address) URIAddr() string { _, host, err := manet.DialArgs(a.ma) - if err != nil { - // the only correct way to construct Address is AddressFromString - // which makes this error appear unexpected - panic(fmt.Errorf("could not get host addr: %w", err)) - } + // the only correct way to construct Address is AddressFromString + // which makes this error appear unexpected + assert.NoError(err, "could not get host addr") if !a.IsTLSEnabled() { return host From e45382b0c18ef6f5b116df4cd89f7813d271d54d Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 8 Apr 2025 16:55:56 +0300 Subject: [PATCH 61/94] [#1693] util: Replace conditional panics with asserts Change-Id: I13b566cde3e6d43d8a75aa2e9b28e63b597adff9 Signed-off-by: Ekaterina Lebedeva --- pkg/util/keyer/dashboard.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/util/keyer/dashboard.go b/pkg/util/keyer/dashboard.go index b2942b52a..6337039a9 100644 --- a/pkg/util/keyer/dashboard.go +++ b/pkg/util/keyer/dashboard.go @@ -6,6 +6,7 @@ import ( "os" "text/tabwriter" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "github.com/mr-tron/base58" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -104,9 +105,7 @@ func (d Dashboard) PrettyPrint(uncompressed, useHex bool) { func base58ToHex(data string) string { val, err := base58.Decode(data) - if err != nil { - panic("produced incorrect base58 value") - } + assert.NoError(err, "produced incorrect base58 value") return hex.EncodeToString(val) } From 36fb15b9a4b1b7771b2011c745222326ee6e9299 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 16 Apr 2025 11:55:00 +0300 Subject: [PATCH 62/94] [#1689] engine: Return error if object is locked during inhume Return `object is locked` error if object doesn't exists but is locked, since the locked index may be populated even when the object itself doesn't exist. Change-Id: If1a145c6efead9873acd33bb4fd22cf6175cbabd Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/inhume.go | 11 ++-- .../engine/inhume_test.go | 55 +++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 6ec9a4ef0..e5a2396e1 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -186,10 +186,6 @@ func (e *StorageEngine) findShards(ctx context.Context, addr oid.Address, checkL default: } - if !objectExists { - return - } - if checkLocked { if isLocked, err := sh.IsLocked(ctx, addr); err != nil { e.log.Warn(ctx, logs.EngineRemovingAnObjectWithoutFullLockingCheck, @@ -202,6 +198,13 @@ func (e *StorageEngine) findShards(ctx context.Context, addr oid.Address, checkL } } + // This exit point must come after checking if the object is locked, + // since the locked index may be populated even if the object doesn't + // exist. + if !objectExists { + return + } + ids = append(ids, sh.ID().String()) // Continue if it's a root object. diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 10cebfb52..0601a43f2 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -242,3 +242,58 @@ func benchmarkInhumeMultipart(b *testing.B, numShards, numObjects int) { b.StopTimer() } } + +func TestInhumeIfObjectDoesntExist(t *testing.T) { + t.Run("object is locked", func(t *testing.T) { + t.Run("inhume without tombstone", func(t *testing.T) { + testInhumeLockedIfObjectDoesntExist(t, false, false) + }) + t.Run("inhume with tombstone", func(t *testing.T) { + testInhumeLockedIfObjectDoesntExist(t, true, false) + }) + t.Run("force inhume without tombstone", func(t *testing.T) { + testInhumeLockedIfObjectDoesntExist(t, false, true) + }) + t.Run("force inhume with tombstone", func(t *testing.T) { + testInhumeLockedIfObjectDoesntExist(t, true, true) + }) + }) +} + +func testInhumeLockedIfObjectDoesntExist(t *testing.T, withTombstone, withForce bool) { + t.Parallel() + + var ( + errLocked *apistatus.ObjectLocked + inhumePrm InhumePrm + ctx = context.Background() + container = cidtest.ID() + object = oidtest.Address() + lock = oidtest.ID() + tombstone = oidtest.Address() + ) + object.SetContainer(container) + tombstone.SetContainer(container) + + engine := testNewEngine(t).setShardsNum(t, 4).prepare(t).engine + defer func() { require.NoError(t, engine.Close(ctx)) }() + + err := engine.Lock(ctx, container, lock, []oid.ID{object.Object()}) + require.NoError(t, err) + + if withTombstone { + inhumePrm.WithTarget(tombstone, object) + } else { + inhumePrm.MarkAsGarbage(object) + } + if withForce { + inhumePrm.WithForceRemoval() + } + + err = engine.Inhume(ctx, inhumePrm) + if withForce { + require.NoError(t, err) + } else { + require.ErrorAs(t, err, &errLocked) + } +} From 100eb8b654b55286caa5bf4aa18c358eab12fd26 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 29 Jan 2025 15:39:28 +0300 Subject: [PATCH 63/94] [#1619] logger: Set tags for `node` components Change-Id: I55ffcce9d2a74fdd47621674739b07f2e20199e3 Signed-off-by: Anton Nikiforov --- .../maintenance/zombie/storage_engine.go | 3 ++- cmd/frostfs-node/config.go | 13 ++++----- cmd/frostfs-node/config/logger/config_test.go | 3 +++ cmd/frostfs-node/morph.go | 14 +++++----- cmd/frostfs-node/object.go | 14 +++++----- cmd/frostfs-node/session.go | 3 ++- cmd/frostfs-node/tree.go | 3 ++- config/example/node.env | 2 ++ config/example/node.json | 8 +++++- config/example/node.yaml | 3 +++ docs/storage-node-configuration.md | 16 ++++++++--- .../blobovnicza/blobovnicza.go | 2 +- .../blobovniczatree/concurrency_test.go | 3 ++- .../blobstor/blobovniczatree/exists_test.go | 3 ++- .../blobstor/blobovniczatree/generic_test.go | 6 +++-- .../blobstor/blobovniczatree/option.go | 11 +++++--- .../blobovniczatree/rebuild_failover_test.go | 3 ++- .../blobstor/blobovniczatree/rebuild_test.go | 27 ++++++++++++------- pkg/local_object_storage/blobstor/blobstor.go | 2 +- .../blobstor/fstree/option.go | 3 +-- .../engine/engine_test.go | 3 ++- .../shard/gc_internal_test.go | 3 ++- pkg/local_object_storage/shard/lock_test.go | 3 ++- pkg/local_object_storage/shard/range_test.go | 3 ++- pkg/local_object_storage/shard/shard.go | 2 +- pkg/local_object_storage/shard/shard_test.go | 3 ++- .../writecache/options.go | 3 +-- pkg/services/object/delete/service.go | 2 +- pkg/services/object/get/service.go | 2 +- pkg/services/object/get/v2/service.go | 2 +- pkg/services/object/search/service.go | 2 +- pkg/services/policer/policer.go | 3 --- pkg/services/replicator/replicator.go | 3 --- pkg/services/session/executor.go | 5 +--- pkg/util/logger/logger.go | 1 + pkg/util/logger/tag_string.go | 19 +++++++++++-- pkg/util/logger/tags.go | 27 ++++++++++++++----- 37 files changed, 151 insertions(+), 77 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go index c54b331f3..5be34d502 100644 --- a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go @@ -159,7 +159,8 @@ func getSubStorages(ctx context.Context, sh *shardconfig.Config) []blobstor.SubS blobovniczatree.WithOpenedCacheExpInterval(sub.OpenedCacheExpInterval()), blobovniczatree.WithInitWorkerCount(sub.InitWorkerCount()), blobovniczatree.WithWaitBeforeDropDB(sub.RebuildDropTimeout()), - blobovniczatree.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + blobovniczatree.WithBlobovniczaLogger(logger.NewLoggerWrapper(zap.NewNop())), + blobovniczatree.WithBlobovniczaTreeLogger(logger.NewLoggerWrapper(zap.NewNop())), blobovniczatree.WithObjectSizeLimit(sh.SmallSizeLimit()), } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index f80401b5b..fae1ca1ca 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -891,7 +891,7 @@ func (c *cfg) engineOpts() []engine.Option { opts = append(opts, engine.WithErrorThreshold(c.EngineCfg.errorThreshold), - engine.WithLogger(c.log), + engine.WithLogger(c.log.WithTag(logger.TagEngine)), engine.WithLowMemoryConsumption(c.EngineCfg.lowMem), ) @@ -928,7 +928,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { writecache.WithMaxCacheSize(wcRead.sizeLimit), writecache.WithMaxCacheCount(wcRead.countLimit), writecache.WithNoSync(wcRead.noSync), - writecache.WithLogger(c.log), + writecache.WithLogger(c.log.WithTag(logger.TagWriteCache)), writecache.WithQoSLimiter(shCfg.limiter), ) } @@ -968,7 +968,8 @@ func (c *cfg) getSubstorageOpts(ctx context.Context, shCfg shardCfg) []blobstor. blobovniczatree.WithOpenedCacheExpInterval(sRead.openedCacheExpInterval), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithWaitBeforeDropDB(sRead.rebuildDropTimeout), - blobovniczatree.WithLogger(c.log), + blobovniczatree.WithBlobovniczaLogger(c.log.WithTag(logger.TagBlobovnicza)), + blobovniczatree.WithBlobovniczaTreeLogger(c.log.WithTag(logger.TagBlobovniczaTree)), blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit), } @@ -991,7 +992,7 @@ func (c *cfg) getSubstorageOpts(ctx context.Context, shCfg shardCfg) []blobstor. fstree.WithPerm(sRead.perm), fstree.WithDepth(sRead.depth), fstree.WithNoSync(sRead.noSync), - fstree.WithLogger(c.log), + fstree.WithLogger(c.log.WithTag(logger.TagFSTree)), } if c.metricsCollector != nil { fstreeOpts = append(fstreeOpts, @@ -1023,7 +1024,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID blobstoreOpts := []blobstor.Option{ blobstor.WithCompression(shCfg.compression), blobstor.WithStorages(ss), - blobstor.WithLogger(c.log), + blobstor.WithLogger(c.log.WithTag(logger.TagBlobstor)), } if c.metricsCollector != nil { blobstoreOpts = append(blobstoreOpts, blobstor.WithMetrics(lsmetrics.NewBlobstoreMetrics(c.metricsCollector.Blobstore()))) @@ -1048,7 +1049,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID var sh shardOptsWithID sh.configID = shCfg.id() sh.shOpts = []shard.Option{ - shard.WithLogger(c.log), + shard.WithLogger(c.log.WithTag(logger.TagShard)), shard.WithRefillMetabase(shCfg.refillMetabase), shard.WithRefillMetabaseWorkersCount(shCfg.refillMetabaseWorkersCount), shard.WithMode(shCfg.mode), diff --git a/cmd/frostfs-node/config/logger/config_test.go b/cmd/frostfs-node/config/logger/config_test.go index ffe8ac693..796ad529e 100644 --- a/cmd/frostfs-node/config/logger/config_test.go +++ b/cmd/frostfs-node/config/logger/config_test.go @@ -22,6 +22,9 @@ func TestLoggerSection_Level(t *testing.T) { require.Equal(t, "debug", loggerconfig.Level(c)) require.Equal(t, "journald", loggerconfig.Destination(c)) require.Equal(t, true, loggerconfig.Timestamp(c)) + tags := loggerconfig.Tags(c) + require.Equal(t, "main, morph", tags[0][0]) + require.Equal(t, "debug", tags[0][1]) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index d3c0f7b81..917cf6fc0 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -84,7 +85,7 @@ func initMorphClient(ctx context.Context, c *cfg) { cli, err := client.New(ctx, c.key, client.WithDialTimeout(morphconfig.DialTimeout(c.appCfg)), - client.WithLogger(c.log), + client.WithLogger(c.log.WithTag(logger.TagMorph)), client.WithMetrics(c.metricsCollector.MorphClientMetrics()), client.WithEndpoints(addresses...), client.WithConnLostCallback(func() { @@ -165,6 +166,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { err error subs subscriber.Subscriber ) + log := c.log.WithTag(logger.TagMorph) fromSideChainBlock, err := c.persistate.UInt32(persistateSideChainLastBlockKey) if err != nil { @@ -173,14 +175,14 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { } subs, err = subscriber.New(ctx, &subscriber.Params{ - Log: c.log, + Log: log, StartFromBlock: fromSideChainBlock, Client: c.cfgMorph.client, }) fatalOnErr(err) lis, err := event.NewListener(event.ListenerParams{ - Logger: c.log, + Logger: log, Subscriber: subs, }) fatalOnErr(err) @@ -198,7 +200,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { setNetmapNotificationParser(c, newEpochNotification, func(src *state.ContainedNotificationEvent) (event.Event, error) { res, err := netmapEvent.ParseNewEpoch(src) if err == nil { - c.log.Info(ctx, logs.FrostFSNodeNewEpochEventFromSidechain, + log.Info(ctx, logs.FrostFSNodeNewEpochEventFromSidechain, zap.Uint64("number", res.(netmapEvent.NewEpoch).EpochNumber()), ) } @@ -209,11 +211,11 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { registerNotificationHandlers(c.cfgContainer.scriptHash, lis, c.cfgContainer.parsers, c.cfgContainer.subscribers) registerBlockHandler(lis, func(ctx context.Context, block *block.Block) { - c.log.Debug(ctx, logs.FrostFSNodeNewBlock, zap.Uint32("index", block.Index)) + log.Debug(ctx, logs.FrostFSNodeNewBlock, zap.Uint32("index", block.Index)) err = c.persistate.SetUInt32(persistateSideChainLastBlockKey, block.Index) if err != nil { - c.log.Warn(ctx, logs.FrostFSNodeCantUpdatePersistentState, + log.Warn(ctx, logs.FrostFSNodeCantUpdatePersistentState, zap.String("chain", "side"), zap.Uint32("block_index", block.Index)) } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 527746d26..c33c02b3f 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -31,6 +31,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -217,9 +218,8 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl } remoteReader := objectService.NewRemoteReader(keyStorage, clientConstructor) - pol := policer.New( - policer.WithLogger(c.log), + policer.WithLogger(c.log.WithTag(logger.TagPolicer)), policer.WithKeySpaceIterator(&keySpaceIterator{ng: ls}), policer.WithBuryFunc(buryFn), policer.WithContainerSource(c.cfgObject.cnrSource), @@ -291,7 +291,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa ls := c.cfgObject.cfgLocalStorage.localStorage return replicator.New( - replicator.WithLogger(c.log), + replicator.WithLogger(c.log.WithTag(logger.TagReplicator)), replicator.WithPutTimeout( replicatorconfig.PutTimeout(c.appCfg), ), @@ -348,7 +348,7 @@ func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Trav c.netMapSource, keyStorage, containerSource, - searchsvc.WithLogger(c.log), + searchsvc.WithLogger(c.log.WithTag(logger.TagSearchSvc)), ) } @@ -374,7 +374,7 @@ func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Tra ), coreConstructor, containerSource, - getsvc.WithLogger(c.log)) + getsvc.WithLogger(c.log.WithTag(logger.TagGetSvc))) } func createGetServiceV2(c *cfg, sGet *getsvc.Service, keyStorage *util.KeyStorage) *getsvcV2.Service { @@ -385,7 +385,7 @@ func createGetServiceV2(c *cfg, sGet *getsvc.Service, keyStorage *util.KeyStorag c.netMapSource, c, c.cfgObject.cnrSource, - getsvcV2.WithLogger(c.log), + getsvcV2.WithLogger(c.log.WithTag(logger.TagGetSvc)), ) } @@ -402,7 +402,7 @@ func createDeleteService(c *cfg, keyStorage *util.KeyStorage, sGet *getsvc.Servi cfg: c, }, keyStorage, - deletesvc.WithLogger(c.log), + deletesvc.WithLogger(c.log.WithTag(logger.TagDeleteSvc)), ) } diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index 2f3c9cbfe..fbfe3f5e6 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/persistent" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/temporary" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" sessionGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -55,7 +56,7 @@ func initSessionService(c *cfg) { server := sessionTransportGRPC.New( sessionSvc.NewSignService( &c.key.PrivateKey, - sessionSvc.NewExecutionService(c.privateTokenStore, c.respSvc, c.log), + sessionSvc.NewExecutionService(c.privateTokenStore, c.respSvc, c.log.WithTag(logger.TagSessionSvc)), ), ) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 67d9c9df0..62af45389 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" "google.golang.org/grpc" @@ -56,7 +57,7 @@ func initTreeService(c *cfg) { tree.WithFrostfsidSubjectProvider(c.frostfsidClient), tree.WithNetmapSource(c.netMapSource), tree.WithPrivateKey(&c.key.PrivateKey), - tree.WithLogger(c.log), + tree.WithLogger(c.log.WithTag(logger.TagTreeSvc)), tree.WithStorage(c.cfgObject.cfgLocalStorage.localStorage), tree.WithContainerCacheSize(treeConfig.CacheSize()), tree.WithReplicationTimeout(treeConfig.ReplicationTimeout()), diff --git a/config/example/node.env b/config/example/node.env index 9d054fe78..e7d7a6cc8 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -1,6 +1,8 @@ FROSTFS_LOGGER_LEVEL=debug FROSTFS_LOGGER_DESTINATION=journald FROSTFS_LOGGER_TIMESTAMP=true +FROSTFS_LOGGER_TAGS_0_NAMES="main, morph" +FROSTFS_LOGGER_TAGS_0_LEVEL="debug" FROSTFS_PPROF_ENABLED=true FROSTFS_PPROF_ADDRESS=localhost:6060 diff --git a/config/example/node.json b/config/example/node.json index 110e99ee8..3f7854d98 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -2,7 +2,13 @@ "logger": { "level": "debug", "destination": "journald", - "timestamp": true + "timestamp": true, + "tags": [ + { + "names": "main, morph", + "level": "debug" + } + ] }, "pprof": { "enabled": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index de5eaa133..32f0cba67 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -2,6 +2,9 @@ logger: level: debug # logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" destination: journald # logger destination: one of "stdout" (default), "journald" timestamp: true + tags: + - names: "main, morph" + level: debug systemdnotify: enabled: true diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 1494d2fca..3e770457b 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -112,11 +112,21 @@ Contains logger parameters. ```yaml logger: level: info + tags: + - names: "main, morph" + level: debug ``` -| Parameter | Type | Default value | Description | -|-----------|----------|---------------|---------------------------------------------------------------------------------------------------| -| `level` | `string` | `info` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal` | +| Parameter | Type | Default value | Description | +|-----------|-----------------------------------------------|---------------|---------------------------------------------------------------------------------------------------| +| `level` | `string` | `info` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal` | +| `tags` | list of [tags descriptions](#tags-subsection) | | Array of tags description. | + +## `tags` subsection +| Parameter | Type | Default value | Description | +|-----------|----------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `names` | `string` | | List of components divided by `,`.
Possible values: `main`, `morph`, `grpcsvc`, `ir`, `processor`, `engine`, `blobovnicza`, `blobstor`, `fstree`, `gc`, `shard`, `writecache`, `deletesvc`, `getsvc`, `searchsvc`, `sessionsvc`, `treesvc`, `policer`, `replicator`. | +| `level` | `string` | | Logging level for the components from `names`, overrides default logging level. | # `contracts` section Contains override values for FrostFS side-chain contract hashes. Most of the time contract diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index 08ef8b86c..a6c40f9fa 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -110,7 +110,7 @@ func WithFullSizeLimit(lim uint64) Option { // WithLogger returns an option to specify Blobovnicza's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = l.With(zap.String("component", "Blobovnicza")) + c.log = l } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go index ec9743b57..f87f4a144 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -19,7 +19,8 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) { st := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(10), WithBlobovniczaShallowDepth(1), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 5414140f0..df2b4ffe5 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -19,7 +19,8 @@ func TestExistsInvalidStorageID(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go index d390ecf1d..9244d765c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go @@ -15,7 +15,8 @@ func TestGeneric(t *testing.T) { helper := func(t *testing.T, dir string) common.Storage { return NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), @@ -43,7 +44,8 @@ func TestControl(t *testing.T) { newTree := func(t *testing.T) common.Storage { return NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 2f6d31b4e..5f268b0f2 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -63,10 +63,15 @@ func initConfig(c *cfg) { } } -func WithLogger(l *logger.Logger) Option { +func WithBlobovniczaTreeLogger(log *logger.Logger) Option { return func(c *cfg) { - c.log = l - c.blzOpts = append(c.blzOpts, blobovnicza.WithLogger(l)) + c.log = log + } +} + +func WithBlobovniczaLogger(log *logger.Logger) Option { + return func(c *cfg) { + c.blzOpts = append(c.blzOpts, blobovnicza.WithLogger(log)) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index 8832603c4..4146ef260 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -140,7 +140,8 @@ func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object, mustUpdateStorageID bool) { b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 9c971bfb6..a7a99fec3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -50,7 +50,8 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), WithBlobovniczaShallowWidth(1), // single directory WithBlobovniczaShallowDepth(1), @@ -106,7 +107,8 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), WithBlobovniczaShallowWidth(1), // single directory WithBlobovniczaShallowDepth(1), @@ -160,7 +162,8 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), WithBlobovniczaShallowWidth(1), // single directory WithBlobovniczaShallowDepth(1), @@ -231,7 +234,8 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), WithBlobovniczaShallowWidth(1), // single directory WithBlobovniczaShallowDepth(1), @@ -262,7 +266,8 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, b.Close(context.Background())) b = NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), WithBlobovniczaShallowWidth(1), WithBlobovniczaShallowDepth(1), @@ -304,7 +309,8 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), // 64KB object size limit WithBlobovniczaShallowWidth(5), WithBlobovniczaShallowDepth(2), // depth = 2 @@ -332,7 +338,8 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { b = NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(32*1024), // 32KB object size limit WithBlobovniczaShallowWidth(5), WithBlobovniczaShallowDepth(3), // depth = 3 @@ -374,7 +381,8 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(sourceWidth), WithBlobovniczaShallowDepth(sourceDepth), @@ -415,7 +423,8 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta b = NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(targetWidth), WithBlobovniczaShallowDepth(targetDepth), diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index edb2c1946..ceaf2538a 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -91,7 +91,7 @@ func WithStorages(st []SubStorage) Option { // WithLogger returns option to specify BlobStor's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = l.With(zap.String("component", "BlobStor")) + c.log = l } } diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go index 7155ddcbb..6f2ac87e1 100644 --- a/pkg/local_object_storage/blobstor/fstree/option.go +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -4,7 +4,6 @@ import ( "io/fs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" ) type Option func(*FSTree) @@ -53,6 +52,6 @@ func WithFileCounter(c FileCounter) Option { func WithLogger(l *logger.Logger) Option { return func(f *FSTree) { - f.log = l.With(zap.String("component", "FSTree")) + f.log = l } } diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 711a76100..fc6d9ee9c 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -116,7 +116,8 @@ func newStorages(t testing.TB, root string, smallSize uint64) []blobstor.SubStor blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1), blobovniczatree.WithPermissions(0o700), - blobovniczatree.WithLogger(test.NewLogger(t))), + blobovniczatree.WithBlobovniczaLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaTreeLogger(test.NewLogger(t))), Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < smallSize }, diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index 9998bbae2..54d2f1510 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -37,7 +37,8 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { { Storage: blobovniczatree.NewBlobovniczaTree( context.Background(), - blobovniczatree.WithLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaTreeLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 5caf3641f..3878a65cd 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -28,9 +28,10 @@ func TestShard_Lock(t *testing.T) { var sh *Shard rootPath := t.TempDir() + l := logger.NewLoggerWrapper(zap.NewNop()) opts := []Option{ WithID(NewIDFromBytes([]byte{})), - WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + WithLogger(l), WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ { diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 146e834cc..06fe9f511 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -79,7 +79,8 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { { Storage: blobovniczatree.NewBlobovniczaTree( context.Background(), - blobovniczatree.WithLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaTreeLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 304a6bf9d..d89b56266 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -205,7 +205,7 @@ func WithPiloramaOptions(opts ...pilorama.Option) Option { func WithLogger(l *logger.Logger) Option { return func(c *cfg) { c.log = l - c.gcCfg.log = l + c.gcCfg.log = l.WithTag(logger.TagGC) } } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index f9ee34488..84be71c4d 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -60,7 +60,8 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard { Storage: blobovniczatree.NewBlobovniczaTree( context.Background(), - blobovniczatree.WithLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaTreeLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(o.rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index dbbe66c19..a4f98ad06 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -5,7 +5,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" ) // Option represents write-cache configuration option. @@ -46,7 +45,7 @@ type options struct { // WithLogger sets logger. func WithLogger(log *logger.Logger) Option { return func(o *options) { - o.log = log.With(zap.String("component", "WriteCache")) + o.log = log } } diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index 867d3f4ef..1c4d7d585 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -92,6 +92,6 @@ func New(gs *getsvc.Service, // WithLogger returns option to specify Delete service's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = l.With(zap.String("component", "objectSDK.Delete service")) + c.log = l } } diff --git a/pkg/services/object/get/service.go b/pkg/services/object/get/service.go index 9ec10b5f2..a103f5a7f 100644 --- a/pkg/services/object/get/service.go +++ b/pkg/services/object/get/service.go @@ -53,6 +53,6 @@ func New( // WithLogger returns option to specify Get service's logger. func WithLogger(l *logger.Logger) Option { return func(s *Service) { - s.log = l.With(zap.String("component", "Object.Get service")) + s.log = l } } diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index fc483b74b..0ec8912fd 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -145,6 +145,6 @@ func (s *Service) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = l.With(zap.String("component", "Object.Get V2 service")) + c.log = l } } diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index e1aeca957..56fe56468 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -94,6 +94,6 @@ func New(e *engine.StorageEngine, // WithLogger returns option to specify Get service's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = l.With(zap.String("component", "Object.Search service")) + c.log = l } } diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index d18b71a23..c91e7cc7c 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" - "go.uber.org/zap" ) type objectsInWork struct { @@ -56,8 +55,6 @@ func New(opts ...Option) *Policer { opts[i](c) } - c.log = c.log.With(zap.String("component", "Object Policer")) - cache, err := lru.New[oid.Address, time.Time](int(c.cacheSize)) assert.NoError(err, fmt.Sprintf("could not create LRU cache with %d size", c.cacheSize)) diff --git a/pkg/services/replicator/replicator.go b/pkg/services/replicator/replicator.go index 6910fa5af..a940cef37 100644 --- a/pkg/services/replicator/replicator.go +++ b/pkg/services/replicator/replicator.go @@ -7,7 +7,6 @@ import ( objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" ) // Replicator represents the utility that replicates @@ -45,8 +44,6 @@ func New(opts ...Option) *Replicator { opts[i](c) } - c.log = c.log.With(zap.String("component", "Object Replicator")) - return &Replicator{ cfg: c, } diff --git a/pkg/services/session/executor.go b/pkg/services/session/executor.go index 12b221613..f0591de71 100644 --- a/pkg/services/session/executor.go +++ b/pkg/services/session/executor.go @@ -33,10 +33,7 @@ func NewExecutionService(exec ServiceExecutor, respSvc *response.Service, l *log } func (s *executorSvc) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - s.log.Debug(ctx, logs.ServingRequest, - zap.String("component", "SessionService"), - zap.String("request", "Create"), - ) + s.log.Debug(ctx, logs.ServingRequest, zap.String("request", "Create")) respBody, err := s.exec.Create(ctx, req.GetBody()) if err != nil { diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 276847be1..a1998cb1a 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -238,5 +238,6 @@ func NewLoggerWrapper(z *zap.Logger) *Logger { return &Logger{ z: z.WithOptions(zap.AddCallerSkip(1)), t: TagMain, + c: z.Core(), } } diff --git a/pkg/util/logger/tag_string.go b/pkg/util/logger/tag_string.go index a25b3c445..1b98f2e62 100644 --- a/pkg/util/logger/tag_string.go +++ b/pkg/util/logger/tag_string.go @@ -13,11 +13,26 @@ func _() { _ = x[TagGrpcSvc-3] _ = x[TagIr-4] _ = x[TagProcessor-5] + _ = x[TagEngine-6] + _ = x[TagBlobovnicza-7] + _ = x[TagBlobovniczaTree-8] + _ = x[TagBlobstor-9] + _ = x[TagFSTree-10] + _ = x[TagGC-11] + _ = x[TagShard-12] + _ = x[TagWriteCache-13] + _ = x[TagDeleteSvc-14] + _ = x[TagGetSvc-15] + _ = x[TagSearchSvc-16] + _ = x[TagSessionSvc-17] + _ = x[TagTreeSvc-18] + _ = x[TagPolicer-19] + _ = x[TagReplicator-20] } -const _Tag_name = "mainmorphgrpc_svcirprocessor" +const _Tag_name = "mainmorphgrpcsvcirprocessorengineblobovniczablobovniczatreeblobstorfstreegcshardwritecachedeletesvcgetsvcsearchsvcsessionsvctreesvcpolicerreplicator" -var _Tag_index = [...]uint8{0, 4, 9, 17, 19, 28} +var _Tag_index = [...]uint8{0, 4, 9, 16, 18, 27, 33, 44, 59, 67, 73, 75, 80, 90, 99, 105, 114, 124, 131, 138, 148} func (i Tag) String() string { i -= 1 diff --git a/pkg/util/logger/tags.go b/pkg/util/logger/tags.go index 51a6f62da..a5386707e 100644 --- a/pkg/util/logger/tags.go +++ b/pkg/util/logger/tags.go @@ -13,12 +13,27 @@ import ( type Tag uint8 const ( - _ Tag = iota // - TagMain // main - TagMorph // morph - TagGrpcSvc // grpc_svc - TagIr // ir - TagProcessor // processor + _ Tag = iota // + TagMain // main + TagMorph // morph + TagGrpcSvc // grpcsvc + TagIr // ir + TagProcessor // processor + TagEngine // engine + TagBlobovnicza // blobovnicza + TagBlobovniczaTree // blobovniczatree + TagBlobstor // blobstor + TagFSTree // fstree + TagGC // gc + TagShard // shard + TagWriteCache // writecache + TagDeleteSvc // deletesvc + TagGetSvc // getsvc + TagSearchSvc // searchsvc + TagSessionSvc // sessionsvc + TagTreeSvc // treesvc + TagPolicer // policer + TagReplicator // replicator defaultLevel = zapcore.InfoLevel ) From 86264e4e20af3eb9f90ae65487a5b5e91eb8a28d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 4 Feb 2025 09:11:36 +0300 Subject: [PATCH 64/94] [#1619] logger: Add benchmark Change-Id: I49e90e8a3689a755755afd0638b327a6b1884795 Signed-off-by: Anton Nikiforov --- pkg/util/logger/logger_test.go | 118 +++++++++++++++++++++++++++++ pkg/util/logger/logger_test.result | 46 +++++++++++ 2 files changed, 164 insertions(+) create mode 100644 pkg/util/logger/logger_test.go create mode 100644 pkg/util/logger/logger_test.result diff --git a/pkg/util/logger/logger_test.go b/pkg/util/logger/logger_test.go new file mode 100644 index 000000000..b867ee6cc --- /dev/null +++ b/pkg/util/logger/logger_test.go @@ -0,0 +1,118 @@ +package logger + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func BenchmarkLogger(b *testing.B) { + ctx := context.Background() + m := map[string]Prm{} + + prm := Prm{} + require.NoError(b, prm.SetLevelString("debug")) + m["logging enabled"] = prm + + prm = Prm{} + require.NoError(b, prm.SetLevelString("error")) + m["logging disabled"] = prm + + prm = Prm{} + require.NoError(b, prm.SetLevelString("error")) + require.NoError(b, prm.SetTags([][]string{{"main", "debug"}, {"morph", "debug"}})) + m["logging enabled via tags"] = prm + + prm = Prm{} + require.NoError(b, prm.SetLevelString("debug")) + require.NoError(b, prm.SetTags([][]string{{"main", "error"}, {"morph", "debug"}})) + m["logging disabled via tags"] = prm + + for k, v := range m { + b.Run(k, func(b *testing.B) { + logger, err := createLogger(v) + require.NoError(b, err) + UpdateLevelForTags(v) + b.ResetTimer() + b.ReportAllocs() + for range b.N { + logger.Info(ctx, "test info") + } + }) + } +} + +type testCore struct { + core zapcore.Core +} + +func (c *testCore) Enabled(lvl zapcore.Level) bool { + return c.core.Enabled(lvl) +} + +func (c *testCore) With(fields []zapcore.Field) zapcore.Core { + c.core = c.core.With(fields) + return c +} + +func (c *testCore) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + return ce.AddCore(e, c) +} + +func (c *testCore) Write(zapcore.Entry, []zapcore.Field) error { + return nil +} + +func (c *testCore) Sync() error { + return c.core.Sync() +} + +func createLogger(prm Prm) (*Logger, error) { + prm.Options = []zap.Option{zap.WrapCore(func(core zapcore.Core) zapcore.Core { + tc := testCore{core: core} + return &tc + })} + return NewLogger(prm) +} + +func TestLoggerOutput(t *testing.T) { + obs, logs := observer.New(zap.NewAtomicLevelAt(zap.DebugLevel)) + + prm := Prm{} + require.NoError(t, prm.SetLevelString("debug")) + prm.Options = []zap.Option{zap.WrapCore(func(zapcore.Core) zapcore.Core { + return obs + })} + loggerMain, err := NewLogger(prm) + require.NoError(t, err) + UpdateLevelForTags(prm) + + loggerMainWith := loggerMain.With(zap.String("key", "value")) + + require.Panics(t, func() { + loggerMainWith.WithTag(TagShard) + }) + loggerShard := loggerMain.WithTag(TagShard) + loggerShard = loggerShard.With(zap.String("key1", "value1")) + + loggerMorph := loggerMain.WithTag(TagMorph) + loggerMorph = loggerMorph.With(zap.String("key2", "value2")) + + ctx := context.Background() + loggerMain.Debug(ctx, "main") + loggerMainWith.Debug(ctx, "main with") + loggerShard.Debug(ctx, "shard") + loggerMorph.Debug(ctx, "morph") + + require.Len(t, logs.All(), 4) + require.Len(t, logs.FilterFieldKey("key").All(), 1) + require.Len(t, logs.FilterFieldKey("key1").All(), 1) + require.Len(t, logs.FilterFieldKey("key2").All(), 1) + require.Len(t, logs.FilterField(zap.String("tag", TagMain.String())).All(), 2) + require.Len(t, logs.FilterField(zap.String("tag", TagShard.String())).All(), 1) + require.Len(t, logs.FilterField(zap.String("tag", TagMorph.String())).All(), 1) +} diff --git a/pkg/util/logger/logger_test.result b/pkg/util/logger/logger_test.result new file mode 100644 index 000000000..612fa2967 --- /dev/null +++ b/pkg/util/logger/logger_test.result @@ -0,0 +1,46 @@ +goos: linux +goarch: amd64 +pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger +cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz +BenchmarkLogger/logging_enabled-8 10000 1156 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1124 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1106 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1096 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1071 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1081 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1074 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1134 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1123 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1144 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.15 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.54 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.22 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.22 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 17.01 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.31 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.61 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.17 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.26 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 21.02 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1146 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1086 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1113 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1157 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1069 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1073 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1096 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1092 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1060 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1153 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.23 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.39 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.47 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.62 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.53 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.53 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.74 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.20 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 17.06 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.60 ns/op 0 B/op 0 allocs/op +PASS +ok git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger 0.260s From cf48069fd8a12df6c0013d60cc0ce0a89420241b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:26:20 +0300 Subject: [PATCH 65/94] [#1718] linter: Resolve gocritic's appendAssign linter See https://go-critic.com/overview#appendassign for details. Change-Id: I991979ea680af25e2cec9097fa12b1c4eebc6c1d Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/util.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index 3955f8ee1..8e4e8b287 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -262,13 +262,8 @@ func OpenSessionViaClient(cmd *cobra.Command, dst SessionPrm, cli *client.Client if _, ok := dst.(*internal.DeleteObjectPrm); ok { common.PrintVerbose(cmd, "Collecting relatives of the removal object...") - rels := collectObjectRelatives(cmd, cli, cnr, *obj) - - if len(rels) == 0 { - objs = []oid.ID{*obj} - } else { - objs = append(rels, *obj) - } + objs = collectObjectRelatives(cmd, cli, cnr, *obj) + objs = append(objs, *obj) } } From 2075e09cedad3150a91216524c014846edb3cb71 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:28:21 +0300 Subject: [PATCH 66/94] [#1718] linter: Resolve gocritic's unslice linter See https://go-critic.com/overview#unslice for details. Change-Id: I6d21e8ce1c9bae56099dc203f5080b0e3ea0c1ef Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-adm/internal/modules/morph/config/config.go | 2 +- cmd/frostfs-lens/internal/schema/writecache/parsers.go | 2 +- pkg/local_object_storage/metabase/delete.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index f64cb4817..c17fb62ff 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -63,7 +63,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig, netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig: nbuf := make([]byte, 8) - copy(nbuf[:], v) + copy(nbuf, v) n := binary.LittleEndian.Uint64(nbuf) _, _ = tw.Write(fmt.Appendf(nil, "%s:\t%d (int)\n", k, n)) case netmap.HomomorphicHashingDisabledKey, netmap.MaintenanceModeAllowedConfig: diff --git a/cmd/frostfs-lens/internal/schema/writecache/parsers.go b/cmd/frostfs-lens/internal/schema/writecache/parsers.go index 7d70b27b2..3bfe2608b 100644 --- a/cmd/frostfs-lens/internal/schema/writecache/parsers.go +++ b/cmd/frostfs-lens/internal/schema/writecache/parsers.go @@ -57,7 +57,7 @@ func DefaultRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, r.addr.SetContainer(cnr) r.addr.SetObject(obj) - r.data = value[:] + r.data = value return &r, nil, nil } diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index cf1d563e9..9a5a6e574 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -363,12 +363,12 @@ func (db *DB) deleteObject( func parentLength(tx *bbolt.Tx, addr oid.Address) int { bucketName := make([]byte, bucketKeySize) - bkt := tx.Bucket(parentBucketName(addr.Container(), bucketName[:])) + bkt := tx.Bucket(parentBucketName(addr.Container(), bucketName)) if bkt == nil { return 0 } - lst, err := decodeList(bkt.Get(objectKey(addr.Object(), bucketName[:]))) + lst, err := decodeList(bkt.Get(objectKey(addr.Object(), bucketName))) if err != nil { return 0 } From d2114759aaaf4e99aa83c1185c0a5d184a83b97a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:38:06 +0300 Subject: [PATCH 67/94] [#1718] linter: Resolve gocritic's typeSwitchVar linter See https://go-critic.com/overview#typeswitchvar for details Change-Id: Ic29db32c9b080576ab51dd484b4376114e9e775c Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/tui/ui.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-lens/internal/tui/ui.go b/cmd/frostfs-lens/internal/tui/ui.go index bd7540b01..cc6b7859e 100644 --- a/cmd/frostfs-lens/internal/tui/ui.go +++ b/cmd/frostfs-lens/internal/tui/ui.go @@ -460,11 +460,11 @@ func (ui *UI) handleInputOnSearching(event *tcell.EventKey) { return } - switch ui.mountedPage.(type) { + switch v := ui.mountedPage.(type) { case *BucketsView: ui.moveNextPage(NewBucketsView(ui, res)) case *RecordsView: - bucket := ui.mountedPage.(*RecordsView).bucket + bucket := v.bucket ui.moveNextPage(NewRecordsView(ui, bucket, res)) } From 8d499f03fee98825d183d18a7ef492a6a019b25a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:40:31 +0300 Subject: [PATCH 68/94] [#1718] linter: Resolve gocritic's elseif linter See https://go-critic.com/overview#elseif for details. Change-Id: I8fd3edfacaeea2b0a83917575d545af7e7ab4d13 Signed-off-by: Dmitrii Stepanov --- pkg/services/container/ape.go | 6 ++---- pkg/services/object/common/target/target.go | 6 ++---- pkg/services/object_manager/tombstone/checker.go | 6 ++---- pkg/services/tree/service.go | 6 ++---- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index e1fbe3960..01bd825d7 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -655,10 +655,8 @@ func (ac *apeChecker) namespaceByOwner(ctx context.Context, owner *refs.OwnerID) subject, err := ac.frostFSIDClient.GetSubject(ctx, addr) if err == nil { namespace = subject.Namespace - } else { - if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { - return "", fmt.Errorf("get subject error: %w", err) - } + } else if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { + return "", fmt.Errorf("get subject error: %w", err) } return namespace, nil } diff --git a/pkg/services/object/common/target/target.go b/pkg/services/object/common/target/target.go index b2ae79dbc..f2bd907db 100644 --- a/pkg/services/object/common/target/target.go +++ b/pkg/services/object/common/target/target.go @@ -89,10 +89,8 @@ func newTrustedTarget(ctx context.Context, prm *objectwriter.Params) (transforme if !ownerObj.Equals(ownerSession) { return nil, fmt.Errorf("session token is missing but object owner id (%s) is different from the default key (%s)", ownerObj, ownerSession) } - } else { - if !ownerObj.Equals(sessionInfo.Owner) { - return nil, fmt.Errorf("different token issuer and object owner identifiers %s/%s", sessionInfo.Owner, ownerObj) - } + } else if !ownerObj.Equals(sessionInfo.Owner) { + return nil, fmt.Errorf("different token issuer and object owner identifiers %s/%s", sessionInfo.Owner, ownerObj) } if prm.SignRequestPrivateKey == nil { diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index a4e36c2dc..e5f001d5a 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -61,10 +61,8 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr logs.TombstoneCouldNotGetTheTombstoneTheSource, zap.Error(err), ) - } else { - if ts != nil { - return g.handleTS(ctx, addrStr, ts, epoch) - } + } else if ts != nil { + return g.handleTS(ctx, addrStr, ts, epoch) } // requested tombstone not diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index a00c8c1cd..5349dd13e 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -436,10 +436,8 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid } if ms == nil { ms = m.Items - } else { - if len(m.Items) != 1 { - return status.Error(codes.InvalidArgument, "multiple non-internal nodes provided") - } + } else if len(m.Items) != 1 { + return status.Error(codes.InvalidArgument, "multiple non-internal nodes provided") } ts = append(ts, m.Time) ps = append(ps, p) From 64900f87e1872dea2794b02ddbc888f171a24a77 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:44:00 +0300 Subject: [PATCH 69/94] [#1718] linter: Resolve gocritic's singleCaseSwitch linter See https://go-critic.com/overview#singlecaseswitch for details. Change-Id: Ied7885f83b4116969771de6f91bc5e1e3b2a4f1e Signed-off-by: Dmitrii Stepanov --- .../blobstor/fstree/fstree_write_generic.go | 9 +++------ pkg/services/control/ir/server/server.go | 3 +-- pkg/util/http/server.go | 3 +-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 07a618b0a..6d633dad6 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -67,12 +67,9 @@ func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error { err := w.writeFile(tmpPath, data) if err != nil { var pe *fs.PathError - if errors.As(err, &pe) { - switch pe.Err { - case syscall.ENOSPC: - err = common.ErrNoSpace - _ = os.RemoveAll(tmpPath) - } + if errors.As(err, &pe) && errors.Is(pe.Err, syscall.ENOSPC) { + err = common.ErrNoSpace + _ = os.RemoveAll(tmpPath) } return err } diff --git a/pkg/services/control/ir/server/server.go b/pkg/services/control/ir/server/server.go index c2a4f88a6..0cfca71c1 100644 --- a/pkg/services/control/ir/server/server.go +++ b/pkg/services/control/ir/server/server.go @@ -35,8 +35,7 @@ func panicOnPrmValue(n string, v any) { // the parameterized private key. func New(prm Prm, netmapClient *netmap.Client, containerClient *container.Client, opts ...Option) *Server { // verify required parameters - switch { - case prm.healthChecker == nil: + if prm.healthChecker == nil { panicOnPrmValue("health checker", prm.healthChecker) } diff --git a/pkg/util/http/server.go b/pkg/util/http/server.go index 923412a7f..2589ab786 100644 --- a/pkg/util/http/server.go +++ b/pkg/util/http/server.go @@ -76,8 +76,7 @@ func New(prm HTTPSrvPrm, opts ...Option) *Server { o(c) } - switch { - case c.shutdownTimeout <= 0: + if c.shutdownTimeout <= 0 { panicOnOptValue("shutdown timeout", c.shutdownTimeout) } From b88fe8c4a7f842c65d98a1edf6b83e54a7f2c3b3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:45:04 +0300 Subject: [PATCH 70/94] [#1718] linter: Resolve gocritic's assignOp linter See https://go-critic.com/overview#assignop for details. Change-Id: I839446846437c8c74c119d8b5669f5b866c247dc Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index d2eef2074..a840275b8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -226,7 +226,7 @@ func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.M func (b *Blobovniczas) addRebuildTempFile(ctx context.Context, path string) (func(), error) { sysPath := filepath.Join(b.rootPath, path) - sysPath = sysPath + rebuildSuffix + sysPath += rebuildSuffix _, err := os.OpenFile(sysPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, b.perm) if err != nil { return nil, err From ca8b01667fac7107a8451843a4b21c656dfc7527 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:46:31 +0300 Subject: [PATCH 71/94] [#1718] linter: Resolve gocritic's unlambda linter See https://go-critic.com/overview#unlambda for details. Change-Id: Iccb2d293ce31a302fcbb2c3f9c55c9b3fa554db5 Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/v2/util.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index bfa7fd619..4b7dcc530 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -182,9 +182,7 @@ func (s *Service) toHashRangePrm(req *objectV2.GetRangeHashRequest) (*getsvc.Ran default: return nil, errUnknownChechsumType(t) case refs.SHA256: - p.SetHashGenerator(func() hash.Hash { - return sha256.New() - }) + p.SetHashGenerator(sha256.New) case refs.TillichZemor: p.SetHashGenerator(func() hash.Hash { return tz.New() From 3cbff575351a9a8ffff384e920d06da56a6c9703 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:47:32 +0300 Subject: [PATCH 72/94] [#1718] linter: Enable gocritic linter See https://go-critic.com/overview#checkers-from-the-diagnostic-group for list of default enabled checkers. `ifElseChain` disabled as it generates doubtful issues. Change-Id: I5937b116d9af8b3cdf8b06451c4904d0b3f67f68 Signed-off-by: Dmitrii Stepanov --- .golangci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 3ac4eb651..e3ec09f60 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -18,6 +18,7 @@ linters: - exhaustive - funlen - gocognit + - gocritic - godot - importas - ineffassign @@ -44,6 +45,9 @@ linters: statements: 60 gocognit: min-complexity: 40 + gocritic: + disabled-checks: + - ifElseChain importas: alias: - pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object From c84013a85408a5403c2cc8f6f9cf98fb7bf79a7b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 15:09:43 +0300 Subject: [PATCH 73/94] [#1719] cli: Fix TTL description Change-Id: I051a27af57a74304713c1f832dc31dbaeb10cbc6 Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/internal/commonflags/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/internal/commonflags/api.go b/cmd/frostfs-cli/internal/commonflags/api.go index 88321176f..6ed21e107 100644 --- a/cmd/frostfs-cli/internal/commonflags/api.go +++ b/cmd/frostfs-cli/internal/commonflags/api.go @@ -9,7 +9,7 @@ const ( TTL = "ttl" TTLShorthand = "" TTLDefault = 2 - TTLUsage = "TTL value in request meta header" + TTLUsage = "The maximum number of intermediate nodes in the request route" XHeadersKey = "xhdr" XHeadersShorthand = "x" From 1a738792c00518eff82b8643135c4bd4a40e0710 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 17 Apr 2025 16:15:12 +0300 Subject: [PATCH 74/94] [#1722] node, ir: Fix documentation for logger tags Close #1722. Change-Id: Iee9b3d96a31353622e1680d2d596ca6a9ee104b4 Signed-off-by: Anton Nikiforov --- config/example/ir.yaml | 2 +- docs/storage-node-configuration.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/example/ir.yaml b/config/example/ir.yaml index a4a006550..ed53f014b 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -4,7 +4,7 @@ logger: level: info # Logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" timestamp: true tags: - - names: "main, morph" # Possible values: `main`, `morph`, `grpc_svc`, `ir`, `processor`. + - names: "main, morph" # Possible values: `main`, `morph`, `grpcsvc`, `ir`, `processor`. level: debug wallet: diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 3e770457b..14ebb53b3 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -123,10 +123,10 @@ logger: | `tags` | list of [tags descriptions](#tags-subsection) | | Array of tags description. | ## `tags` subsection -| Parameter | Type | Default value | Description | -|-----------|----------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `names` | `string` | | List of components divided by `,`.
Possible values: `main`, `morph`, `grpcsvc`, `ir`, `processor`, `engine`, `blobovnicza`, `blobstor`, `fstree`, `gc`, `shard`, `writecache`, `deletesvc`, `getsvc`, `searchsvc`, `sessionsvc`, `treesvc`, `policer`, `replicator`. | -| `level` | `string` | | Logging level for the components from `names`, overrides default logging level. | +| Parameter | Type | Default value | Description | +|-----------|----------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `names` | `string` | | List of components divided by `,`.
Possible values: `main`, `engine`, `blobovnicza`, `blobovniczatree`, `blobstor`, `fstree`, `gc`, `shard`, `writecache`, `deletesvc`, `getsvc`, `searchsvc`, `sessionsvc`, `treesvc`, `policer`, `replicator`. | +| `level` | `string` | | Logging level for the components from `names`, overrides default logging level. | # `contracts` section Contains override values for FrostFS side-chain contract hashes. Most of the time contract From 6567ceaf132c3295b8f011920d3718c989168b2f Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 17 Apr 2025 13:20:18 +0300 Subject: [PATCH 75/94] [#1702] tree: Support reloading 'tree.authorized_keys' on SIGHUP Allows updating the list of public keys authorized to invoke 'Tree service' operations without restarting the node. Change-Id: I01b6e05875b7ae3f3218062eb12bf9755e87f2a3 Signed-off-by: Alexander Chuprov --- cmd/frostfs-node/config.go | 7 +++++++ pkg/services/tree/options.go | 8 +++----- pkg/services/tree/service.go | 14 ++++++++++++++ pkg/services/tree/signature.go | 4 ++-- pkg/services/tree/signature_test.go | 1 + 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index fae1ca1ca..b688acfde 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -30,6 +30,7 @@ import ( objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" + treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" @@ -1403,6 +1404,12 @@ func (c *cfg) getComponents(ctx context.Context) []dCmp { } return err }}) + if c.treeService != nil { + components = append(components, dCmp{"tree", func() error { + c.treeService.ReloadAuthorizedKeys(treeconfig.Tree(c.appCfg).AuthorizedKeys()) + return nil + }}) + } if cmp, updated := metricsComponent(c); updated { if cmp.enabled { cmp.preReload = enableMetricsSvc diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index 4ad760846..56cbcc081 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -3,6 +3,7 @@ package tree import ( "context" "crypto/ecdsa" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" @@ -41,7 +42,7 @@ type cfg struct { replicatorWorkerCount int replicatorTimeout time.Duration containerCacheSize int - authorizedKeys [][]byte + authorizedKeys atomic.Pointer[[][]byte] syncBatchSize int localOverrideStorage policyengine.LocalOverrideStorage @@ -147,10 +148,7 @@ func WithMetrics(v MetricsRegister) Option { // keys that have rights to use Tree service. func WithAuthorizedKeys(keys keys.PublicKeys) Option { return func(c *cfg) { - c.authorizedKeys = nil - for _, key := range keys { - c.authorizedKeys = append(c.authorizedKeys, key.Bytes()) - } + c.authorizedKeys.Store(fromPublicKeys(keys)) } } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 5349dd13e..a4bc0c97c 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -17,6 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/panjf2000/ants/v2" "go.uber.org/zap" "google.golang.org/grpc/codes" @@ -59,6 +60,7 @@ func New(opts ...Option) *Service { s.replicatorTimeout = defaultReplicatorSendTimeout s.syncBatchSize = defaultSyncBatchSize s.metrics = defaultMetricsRegister{} + s.authorizedKeys.Store(&[][]byte{}) for i := range opts { opts[i](&s.cfg) @@ -782,3 +784,15 @@ func (s *Service) Healthcheck(context.Context, *HealthcheckRequest) (*Healthchec return new(HealthcheckResponse), nil } + +func (s *Service) ReloadAuthorizedKeys(newKeys keys.PublicKeys) { + s.authorizedKeys.Store(fromPublicKeys(newKeys)) +} + +func fromPublicKeys(keys keys.PublicKeys) *[][]byte { + buff := make([][]byte, len(keys)) + for i, k := range keys { + buff[i] = k.Bytes() + } + return &buff +} diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 89056056d..8221a4546 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -95,8 +95,8 @@ func (s *Service) isAuthorized(req message, op acl.Op) (bool, error) { } key := sign.GetKey() - for i := range s.authorizedKeys { - if bytes.Equal(s.authorizedKeys[i], key) { + for _, currentKey := range *s.authorizedKeys.Load() { + if bytes.Equal(currentKey, key) { return true, nil } } diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 947de8e58..f5659d5e2 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -152,6 +152,7 @@ func TestMessageSign(t *testing.T) { apeChecker: checkercore.New(e.LocalStorage(), e.MorphRuleChainStorage(), frostfsidProvider, dummyEpochSource{}), } + s.cfg.authorizedKeys.Store(&[][]byte{}) rawCID1 := make([]byte, sha256.Size) cid1.Encode(rawCID1) From 4bcb67263076baebb2192c49c4d9c6e8d46627cb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Apr 2025 18:28:36 +0300 Subject: [PATCH 76/94] [#1707] tree: Pass tracing context in `forEachNode` Change-Id: I884dcd215bd3934f9b4ea43dcc77e38f9dadcf10 Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/redirect.go | 14 +++++++------- pkg/services/tree/service.go | 8 ++++---- pkg/services/tree/sync.go | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 3dcdc4fc7..647f8cb30 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -19,8 +19,8 @@ var errNoSuitableNode = errors.New("no node was found to execute the request") func relayUnary[Req any, Resp any](ctx context.Context, s *Service, ns []netmapSDK.NodeInfo, req *Req, callback func(TreeServiceClient, context.Context, *Req, ...grpc.CallOption) (*Resp, error)) (*Resp, error) { var resp *Resp var outErr error - err := s.forEachNode(ctx, ns, func(c TreeServiceClient) bool { - resp, outErr = callback(c, ctx, req) + err := s.forEachNode(ctx, ns, func(fCtx context.Context, c TreeServiceClient) bool { + resp, outErr = callback(c, fCtx, req) return true }) if err != nil { @@ -31,7 +31,7 @@ func relayUnary[Req any, Resp any](ctx context.Context, s *Service, ns []netmapS // forEachNode executes callback for each node in the container until true is returned. // Returns errNoSuitableNode if there was no successful attempt to dial any node. -func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo, f func(c TreeServiceClient) bool) error { +func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo, f func(context.Context, TreeServiceClient) bool) error { for _, n := range cntNodes { if bytes.Equal(n.PublicKey(), s.rawPub) { return nil @@ -42,9 +42,9 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo for _, n := range cntNodes { var stop bool for endpoint := range n.NetworkEndpoints() { - stop = s.execOnClient(ctx, endpoint, func(c TreeServiceClient) bool { + stop = s.execOnClient(ctx, endpoint, func(fCtx context.Context, c TreeServiceClient) bool { called = true - return f(c) + return f(fCtx, c) }) if called { break @@ -60,7 +60,7 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo return nil } -func (s *Service) execOnClient(ctx context.Context, endpoint string, f func(TreeServiceClient) bool) bool { +func (s *Service) execOnClient(ctx context.Context, endpoint string, f func(context.Context, TreeServiceClient) bool) bool { ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.IterateNetworkEndpoints", trace.WithAttributes( attribute.String("endpoint", endpoint), @@ -73,5 +73,5 @@ func (s *Service) execOnClient(ctx context.Context, endpoint string, f func(Tree } s.log.Debug(ctx, logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint)) - return f(c) + return f(ctx, c) } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index a4bc0c97c..3994d6973 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -391,8 +391,8 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS if pos < 0 { var cli TreeService_GetSubTreeClient var outErr error - err = s.forEachNode(srv.Context(), ns, func(c TreeServiceClient) bool { - cli, outErr = c.GetSubTree(srv.Context(), req) + err = s.forEachNode(srv.Context(), ns, func(fCtx context.Context, c TreeServiceClient) bool { + cli, outErr = c.GetSubTree(fCtx, req) return true }) if err != nil { @@ -655,8 +655,8 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) if pos < 0 { var cli TreeService_GetOpLogClient var outErr error - err := s.forEachNode(srv.Context(), ns, func(c TreeServiceClient) bool { - cli, outErr = c.GetOpLog(srv.Context(), req) + err := s.forEachNode(srv.Context(), ns, func(fCtx context.Context, c TreeServiceClient) bool { + cli, outErr = c.GetOpLog(fCtx, req) return true }) if err != nil { diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 32297f9d7..d4040337d 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -76,8 +76,8 @@ func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { var treesToSync []string var outErr error - err = s.forEachNode(ctx, nodes, func(c TreeServiceClient) bool { - resp, outErr = c.TreeList(ctx, req) + err = s.forEachNode(ctx, nodes, func(fCtx context.Context, c TreeServiceClient) bool { + resp, outErr = c.TreeList(fCtx, req) if outErr != nil { return false } From a27e0035083b8e66ee446c949f98405c1ac2791d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 21 Apr 2025 10:53:51 +0300 Subject: [PATCH 77/94] [#1709] qos: Add context.Done check before schedule request Do not push request to schedule queue, if context is already cancelled. Change-Id: Ieef837a7d423158e3dbb0c3b4efecaa20744c845 Signed-off-by: Dmitrii Stepanov --- internal/qos/limiter.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index 5851d7626..2d7de32fc 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -149,6 +149,11 @@ func (n *mClockLimiter) WriteRequest(ctx context.Context) (ReleaseFunc, error) { } func requestArrival(ctx context.Context, s scheduler, stats map[string]*stat) (ReleaseFunc, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } tag, ok := tagging.IOTagFromContext(ctx) if !ok { tag = IOTagClient.String() From 3a441f072ffb955c6513eb7dd7733da311590ad7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 21 Apr 2025 12:13:32 +0300 Subject: [PATCH 78/94] [#1709] shard: Check if context canceled for shard iteration If context has already been canceled, then there is no need to check other shards. At the same time, it is necessary to avoid handling context cancellation in each handler. Therefore, the context check has been moved to the shard iteration method, which now returns an error. Change-Id: I70030ace36593ce7d2b8376bee39fe82e9dbf88f Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/container.go | 27 ++++--- pkg/local_object_storage/engine/delete.go | 12 +-- pkg/local_object_storage/engine/exists.go | 6 +- pkg/local_object_storage/engine/get.go | 16 ++-- pkg/local_object_storage/engine/head.go | 6 +- pkg/local_object_storage/engine/inhume.go | 77 ++++++++++++-------- pkg/local_object_storage/engine/lock.go | 16 +++- pkg/local_object_storage/engine/put.go | 6 +- pkg/local_object_storage/engine/range.go | 16 ++-- pkg/local_object_storage/engine/select.go | 30 +++++--- pkg/local_object_storage/engine/shards.go | 22 +++++- 12 files changed, 149 insertions(+), 86 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index dedc7e12c..626372f43 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -198,6 +198,7 @@ const ( EngineInterruptProcessingTheExpiredLocks = "interrupt processing the expired locks" EngineInterruptGettingLockers = "can't get object's lockers" EngineInterruptProcessingTheDeletedLocks = "interrupt processing the deleted locks" + EngineInterruptProcessingTheExpiredTombstones = "interrupt processing the expired tombstones" EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly = "failed to move shard in degraded-read-only mode, moving to read-only" EngineFailedToMoveShardInReadonlyMode = "failed to move shard in read-only mode" EngineShardIsMovedInReadonlyModeDueToErrorThreshold = "shard is moved in read-only mode due to error threshold" diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 03a299b93..e0617a832 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -48,8 +48,9 @@ func (e *StorageEngine) ContainerSize(ctx context.Context, prm ContainerSizePrm) defer elapsed("ContainerSize", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res = e.containerSize(ctx, prm) - return nil + var csErr error + res, csErr = e.containerSize(ctx, prm) + return csErr }) return @@ -69,8 +70,9 @@ func ContainerSize(ctx context.Context, e *StorageEngine, id cid.ID) (uint64, er return res.Size(), nil } -func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) (res ContainerSizeRes) { - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { +func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) (ContainerSizeRes, error) { + var res ContainerSizeRes + err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { var csPrm shard.ContainerSizePrm csPrm.SetContainerID(prm.cnr) @@ -86,7 +88,7 @@ func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) return false }) - return + return res, err } // ListContainers returns a unique container IDs presented in the engine objects. @@ -96,8 +98,9 @@ func (e *StorageEngine) ListContainers(ctx context.Context, _ ListContainersPrm) defer elapsed("ListContainers", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res = e.listContainers(ctx) - return nil + var lcErr error + res, lcErr = e.listContainers(ctx) + return lcErr }) return @@ -115,10 +118,10 @@ func ListContainers(ctx context.Context, e *StorageEngine) ([]cid.ID, error) { return res.Containers(), nil } -func (e *StorageEngine) listContainers(ctx context.Context) ListContainersRes { +func (e *StorageEngine) listContainers(ctx context.Context) (ListContainersRes, error) { uniqueIDs := make(map[string]cid.ID) - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { res, err := sh.ListContainers(ctx, shard.ListContainersPrm{}) if err != nil { e.reportShardError(ctx, sh, "can't get list of containers", err) @@ -133,7 +136,9 @@ func (e *StorageEngine) listContainers(ctx context.Context) ListContainersRes { } return false - }) + }); err != nil { + return ListContainersRes{}, err + } result := make([]cid.ID, 0, len(uniqueIDs)) for _, v := range uniqueIDs { @@ -142,5 +147,5 @@ func (e *StorageEngine) listContainers(ctx context.Context) ListContainersRes { return ListContainersRes{ containers: result, - } + }, nil } diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 5e5f65fa2..223cdbc48 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -71,7 +71,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) error { // Removal of a big object is done in multiple stages: // 1. Remove the parent object. If it is locked or already removed, return immediately. // 2. Otherwise, search for all objects with a particular SplitID and delete them too. - e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { + if err := e.iterateOverSortedShards(ctx, prm.addr, func(_ int, sh hashedShard) (stop bool) { var existsPrm shard.ExistsPrm existsPrm.Address = prm.addr @@ -116,20 +116,22 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) error { // If a parent object is removed we should set GC mark on each shard. return splitInfo == nil - }) + }); err != nil { + return err + } if locked.is { return new(apistatus.ObjectLocked) } if splitInfo != nil { - e.deleteChildren(ctx, prm.addr, prm.forceRemoval, splitInfo.SplitID()) + return e.deleteChildren(ctx, prm.addr, prm.forceRemoval, splitInfo.SplitID()) } return nil } -func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, force bool, splitID *objectSDK.SplitID) { +func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, force bool, splitID *objectSDK.SplitID) error { var fs objectSDK.SearchFilters fs.AddSplitIDFilter(objectSDK.MatchStringEqual, splitID) @@ -142,7 +144,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo inhumePrm.ForceRemoval() } - e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { + return e.iterateOverSortedShards(ctx, addr, func(_ int, sh hashedShard) (stop bool) { res, err := sh.Select(ctx, selectPrm) if err != nil { e.log.Warn(ctx, logs.EngineErrorDuringSearchingForObjectChildren, diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index 9d2b1c1b7..7dac9eb97 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -18,7 +18,7 @@ func (e *StorageEngine) exists(ctx context.Context, shPrm shard.ExistsPrm) (bool exists := false locked := false - e.iterateOverSortedShards(shPrm.Address, func(_ int, sh hashedShard) (stop bool) { + if err := e.iterateOverSortedShards(ctx, shPrm.Address, func(_ int, sh hashedShard) (stop bool) { res, err := sh.Exists(ctx, shPrm) if err != nil { if client.IsErrObjectAlreadyRemoved(err) { @@ -50,7 +50,9 @@ func (e *StorageEngine) exists(ctx context.Context, shPrm shard.ExistsPrm) (bool } return false - }) + }); err != nil { + return false, false, err + } if alreadyRemoved { return false, false, new(apistatus.ObjectAlreadyRemoved) diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 74c64bbb6..0694c53f3 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -78,7 +78,9 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { Engine: e, } - it.tryGetWithMeta(ctx) + if err := it.tryGetWithMeta(ctx); err != nil { + return GetRes{}, err + } if it.SplitInfo != nil { return GetRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(it.SplitInfo)) @@ -97,7 +99,9 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { return GetRes{}, it.OutError } - it.tryGetFromBlobstore(ctx) + if err := it.tryGetFromBlobstore(ctx); err != nil { + return GetRes{}, err + } if it.Object == nil { return GetRes{}, it.OutError @@ -133,8 +137,8 @@ type getShardIterator struct { ecInfoErr *objectSDK.ECInfoError } -func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { - i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { +func (i *getShardIterator) tryGetWithMeta(ctx context.Context) error { + return i.Engine.iterateOverSortedShards(ctx, i.Address, func(_ int, sh hashedShard) (stop bool) { noMeta := sh.GetMode().NoMetabase() i.ShardPrm.SetIgnoreMeta(noMeta) @@ -187,13 +191,13 @@ func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { }) } -func (i *getShardIterator) tryGetFromBlobstore(ctx context.Context) { +func (i *getShardIterator) tryGetFromBlobstore(ctx context.Context) error { // If the object is not found but is present in metabase, // try to fetch it from blobstor directly. If it is found in any // blobstor, increase the error counter for the shard which contains the meta. i.ShardPrm.SetIgnoreMeta(true) - i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { + return i.Engine.iterateOverSortedShards(ctx, i.Address, func(_ int, sh hashedShard) (stop bool) { if sh.GetMode().NoMetabase() { // Already visited. return false diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index d6892f129..d436dd411 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -82,7 +82,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) shPrm.SetAddress(prm.addr) shPrm.SetRaw(prm.raw) - e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { + if err := e.iterateOverSortedShards(ctx, prm.addr, func(_ int, sh hashedShard) (stop bool) { shPrm.ShardLooksBad = sh.errorCount.Load() >= e.errorsThreshold res, err := sh.Head(ctx, shPrm) if err != nil { @@ -123,7 +123,9 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) } head = res.Object() return true - }) + }); err != nil { + return HeadRes{}, err + } if head != nil { return HeadRes{head: head}, nil diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index e5a2396e1..e13f04927 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -158,7 +158,7 @@ func (e *StorageEngine) findShards(ctx context.Context, addr oid.Address, checkL objectExists bool ) - e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { + if err := e.iterateOverSortedShards(ctx, addr, func(_ int, sh hashedShard) (stop bool) { objectExists = false prm.Address = addr @@ -209,7 +209,9 @@ func (e *StorageEngine) findShards(ctx context.Context, addr oid.Address, checkL // Continue if it's a root object. return !isRootObject - }) + }); err != nil { + return nil, err + } if retErr != nil { return nil, retErr @@ -229,7 +231,7 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e var err error var outErr error - e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(h hashedShard) (stop bool) { locked, err = h.IsLocked(ctx, addr) if err != nil { e.reportShardError(ctx, h, "can't check object's lockers", err, zap.Stringer("address", addr)) @@ -238,7 +240,9 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e } return locked - }) + }); err != nil { + return false, err + } if locked { return locked, nil @@ -258,7 +262,7 @@ func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.I var allLocks []oid.ID var outErr error - e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(h hashedShard) (stop bool) { locks, err := h.GetLocks(ctx, addr) if err != nil { e.reportShardError(ctx, h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr)) @@ -266,7 +270,9 @@ func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.I } allLocks = append(allLocks, locks...) return false - }) + }); err != nil { + return nil, err + } if len(allLocks) > 0 { return allLocks, nil } @@ -274,20 +280,23 @@ func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.I } func (e *StorageEngine) processExpiredTombstones(ctx context.Context, addrs []meta.TombstonedObject) { - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { sh.HandleExpiredTombstones(ctx, addrs) select { case <-ctx.Done(): + e.log.Info(ctx, logs.EngineInterruptProcessingTheExpiredTombstones, zap.Error(ctx.Err())) return true default: return false } - }) + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingTheExpiredTombstones, zap.Error(err)) + } } func (e *StorageEngine) processExpiredLocks(ctx context.Context, epoch uint64, lockers []oid.Address) { - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { sh.HandleExpiredLocks(ctx, epoch, lockers) select { @@ -297,11 +306,13 @@ func (e *StorageEngine) processExpiredLocks(ctx context.Context, epoch uint64, l default: return false } - }) + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingTheExpiredLocks, zap.Error(err)) + } } func (e *StorageEngine) processDeletedLocks(ctx context.Context, lockers []oid.Address) { - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { sh.HandleDeletedLocks(ctx, lockers) select { @@ -311,26 +322,25 @@ func (e *StorageEngine) processDeletedLocks(ctx context.Context, lockers []oid.A default: return false } - }) + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingTheDeletedLocks, zap.Error(err)) + } } func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid.ID) { if len(ids) == 0 { return } - idMap, err := e.selectNonExistentIDs(ctx, ids) if err != nil { return } - if len(idMap) == 0 { return } - var failed bool var prm shard.ContainerSizePrm - e.iterateOverUnsortedShards(func(sh hashedShard) bool { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) bool { select { case <-ctx.Done(): e.log.Info(ctx, logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(ctx.Err())) @@ -357,13 +367,15 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid } return len(idMap) == 0 - }) - + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(err)) + return + } if failed || len(idMap) == 0 { return } - e.iterateOverUnsortedShards(func(sh hashedShard) bool { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) bool { select { case <-ctx.Done(): e.log.Info(ctx, logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(ctx.Err())) @@ -381,12 +393,13 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid } return false - }) - + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(err)) + return + } if failed { return } - for id := range idMap { e.metrics.DeleteContainerSize(id.EncodeToString()) } @@ -396,19 +409,16 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci if len(ids) == 0 { return } - idMap, err := e.selectNonExistentIDs(ctx, ids) if err != nil { return } - if len(idMap) == 0 { return } - var failed bool var prm shard.ContainerCountPrm - e.iterateOverUnsortedShards(func(sh hashedShard) bool { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) bool { select { case <-ctx.Done(): e.log.Info(ctx, logs.EngineInterruptProcessingZeroCountContainers, zap.Error(ctx.Err())) @@ -435,13 +445,15 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci } return len(idMap) == 0 - }) - + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingZeroCountContainers, zap.Error(err)) + return + } if failed || len(idMap) == 0 { return } - e.iterateOverUnsortedShards(func(sh hashedShard) bool { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) bool { select { case <-ctx.Done(): e.log.Info(ctx, logs.EngineInterruptProcessingZeroCountContainers, zap.Error(ctx.Err())) @@ -459,12 +471,13 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci } return false - }) - + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingZeroCountContainers, zap.Error(err)) + return + } if failed { return } - for id := range idMap { e.metrics.DeleteContainerCount(id.EncodeToString()) } diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 18fff9cad..3b0cf74f9 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -41,11 +41,19 @@ func (e *StorageEngine) Lock(ctx context.Context, idCnr cid.ID, locker oid.ID, l func (e *StorageEngine) lock(ctx context.Context, idCnr cid.ID, locker oid.ID, locked []oid.ID) error { for i := range locked { - switch e.lockSingle(ctx, idCnr, locker, locked[i], true) { + st, err := e.lockSingle(ctx, idCnr, locker, locked[i], true) + if err != nil { + return err + } + switch st { case 1: return logicerr.Wrap(new(apistatus.LockNonRegularObject)) case 0: - switch e.lockSingle(ctx, idCnr, locker, locked[i], false) { + st, err = e.lockSingle(ctx, idCnr, locker, locked[i], false) + if err != nil { + return err + } + switch st { case 1: return logicerr.Wrap(new(apistatus.LockNonRegularObject)) case 0: @@ -61,13 +69,13 @@ func (e *StorageEngine) lock(ctx context.Context, idCnr cid.ID, locker oid.ID, l // - 0: fail // - 1: locking irregular object // - 2: ok -func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8) { +func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8, retErr error) { // code is pretty similar to inhumeAddr, maybe unify? root := false var addrLocked oid.Address addrLocked.SetContainer(idCnr) addrLocked.SetObject(locked) - e.iterateOverSortedShards(addrLocked, func(_ int, sh hashedShard) (stop bool) { + retErr = e.iterateOverSortedShards(ctx, addrLocked, func(_ int, sh hashedShard) (stop bool) { defer func() { // if object is root we continue since information about it // can be presented in other shards diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index b348d13a2..10cf5ffd5 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -96,7 +96,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { } var shRes putToShardRes - e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { + if err := e.iterateOverSortedShards(ctx, addr, func(_ int, sh hashedShard) (stop bool) { e.mtx.RLock() _, ok := e.shards[sh.ID().String()] e.mtx.RUnlock() @@ -106,7 +106,9 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { } shRes = e.putToShard(ctx, sh, addr, prm.Object, prm.IsIndexedContainer) return shRes.status != putToShardUnknown - }) + }); err != nil { + return err + } switch shRes.status { case putToShardUnknown: return errPutShard diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index a468cf594..7ec4742d8 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -93,7 +93,9 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error Engine: e, } - it.tryGetWithMeta(ctx) + if err := it.tryGetWithMeta(ctx); err != nil { + return RngRes{}, err + } if it.SplitInfo != nil { return RngRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(it.SplitInfo)) @@ -109,7 +111,9 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error return RngRes{}, it.OutError } - it.tryGetFromBlobstor(ctx) + if err := it.tryGetFromBlobstor(ctx); err != nil { + return RngRes{}, err + } if it.Object == nil { return RngRes{}, it.OutError @@ -157,8 +161,8 @@ type getRangeShardIterator struct { Engine *StorageEngine } -func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { - i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { +func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) error { + return i.Engine.iterateOverSortedShards(ctx, i.Address, func(_ int, sh hashedShard) (stop bool) { noMeta := sh.GetMode().NoMetabase() i.HasDegraded = i.HasDegraded || noMeta i.ShardPrm.SetIgnoreMeta(noMeta) @@ -209,13 +213,13 @@ func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { }) } -func (i *getRangeShardIterator) tryGetFromBlobstor(ctx context.Context) { +func (i *getRangeShardIterator) tryGetFromBlobstor(ctx context.Context) error { // If the object is not found but is present in metabase, // try to fetch it from blobstor directly. If it is found in any // blobstor, increase the error counter for the shard which contains the meta. i.ShardPrm.SetIgnoreMeta(true) - i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { + return i.Engine.iterateOverSortedShards(ctx, i.Address, func(_ int, sh hashedShard) (stop bool) { if sh.GetMode().NoMetabase() { // Already processed it without a metabase. return false diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index fc8b4a9a7..4243a5481 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -54,14 +54,15 @@ func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRe defer elapsed("Select", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res = e._select(ctx, prm) - return nil + var sErr error + res, sErr = e._select(ctx, prm) + return sErr }) return } -func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) SelectRes { +func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) { addrList := make([]oid.Address, 0) uniqueMap := make(map[string]struct{}) @@ -69,7 +70,7 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) SelectRes { shPrm.SetContainerID(prm.cnr, prm.indexedContainer) shPrm.SetFilters(prm.filters) - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { res, err := sh.Select(ctx, shPrm) if err != nil { e.reportShardError(ctx, sh, "could not select objects from shard", err) @@ -84,11 +85,13 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) SelectRes { } return false - }) + }); err != nil { + return SelectRes{}, err + } return SelectRes{ addrList: addrList, - } + }, nil } // List returns `limit` available physically storage object addresses in engine. @@ -98,20 +101,21 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) SelectRes { func (e *StorageEngine) List(ctx context.Context, limit uint64) (res SelectRes, err error) { defer elapsed("List", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res = e.list(ctx, limit) - return nil + var lErr error + res, lErr = e.list(ctx, limit) + return lErr }) return } -func (e *StorageEngine) list(ctx context.Context, limit uint64) SelectRes { +func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, error) { addrList := make([]oid.Address, 0, limit) uniqueMap := make(map[string]struct{}) ln := uint64(0) // consider iterating over shuffled shards - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { res, err := sh.List(ctx) // consider limit result of shard iterator if err != nil { e.reportShardError(ctx, sh, "could not select objects from shard", err) @@ -130,11 +134,13 @@ func (e *StorageEngine) list(ctx context.Context, limit uint64) SelectRes { } return false - }) + }); err != nil { + return SelectRes{}, err + } return SelectRes{ addrList: addrList, - } + }, nil } // Select selects objects from local storage using provided filters. diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 6e6c08bb5..69067c500 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -280,20 +280,32 @@ func (e *StorageEngine) unsortedShards() []hashedShard { return shards } -func (e *StorageEngine) iterateOverSortedShards(addr oid.Address, handler func(int, hashedShard) (stop bool)) { +func (e *StorageEngine) iterateOverSortedShards(ctx context.Context, addr oid.Address, handler func(int, hashedShard) (stop bool)) error { for i, sh := range e.sortShards(addr) { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } if handler(i, sh) { break } } + return nil } -func (e *StorageEngine) iterateOverUnsortedShards(handler func(hashedShard) (stop bool)) { +func (e *StorageEngine) iterateOverUnsortedShards(ctx context.Context, handler func(hashedShard) (stop bool)) error { for _, sh := range e.unsortedShards() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } if handler(sh) { break } } + return nil } // SetShardMode sets mode of the shard with provided identifier. @@ -433,7 +445,7 @@ func (e *StorageEngine) ListShardsForObject(ctx context.Context, obj oid.Address var siErr *objectSDK.SplitInfoError var ecErr *objectSDK.ECInfoError - e.iterateOverUnsortedShards(func(hs hashedShard) (stop bool) { + if itErr := e.iterateOverUnsortedShards(ctx, func(hs hashedShard) (stop bool) { res, exErr := hs.Exists(ctx, prm) if exErr != nil { if client.IsErrObjectAlreadyRemoved(exErr) { @@ -463,6 +475,8 @@ func (e *StorageEngine) ListShardsForObject(ctx context.Context, obj oid.Address info = append(info, hs.DumpInfo()) } return false - }) + }); itErr != nil { + return nil, itErr + } return info, err } From 908b08108d7781bad2054d449c427389f2dfe9f3 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 21 Apr 2025 11:59:21 +0300 Subject: [PATCH 79/94] [#1689] engine/test: Fix `TestInhumeIfObjectDoesntExist` test Removed an invalid test case which used exclusive options, added object status check after removal. Change-Id: I4551c0e4532fb669ee6c72871dc4bd34707d8469 Signed-off-by: Aleksey Savchuk --- .../engine/inhume_test.go | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 0601a43f2..fa73dcad5 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -11,6 +11,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -251,21 +252,23 @@ func TestInhumeIfObjectDoesntExist(t *testing.T) { t.Run("inhume with tombstone", func(t *testing.T) { testInhumeLockedIfObjectDoesntExist(t, true, false) }) - t.Run("force inhume without tombstone", func(t *testing.T) { + t.Run("force inhume", func(t *testing.T) { testInhumeLockedIfObjectDoesntExist(t, false, true) }) - t.Run("force inhume with tombstone", func(t *testing.T) { - testInhumeLockedIfObjectDoesntExist(t, true, true) - }) }) } func testInhumeLockedIfObjectDoesntExist(t *testing.T, withTombstone, withForce bool) { t.Parallel() + // Due to the tests design it is possible to set both the options, + // however removal with tombstone and force removal are exclusive. + require.False(t, withTombstone && withForce) + var ( errLocked *apistatus.ObjectLocked inhumePrm InhumePrm + headPrm HeadPrm ctx = context.Background() container = cidtest.ID() object = oidtest.Address() @@ -291,9 +294,17 @@ func testInhumeLockedIfObjectDoesntExist(t *testing.T, withTombstone, withForce } err = engine.Inhume(ctx, inhumePrm) - if withForce { - require.NoError(t, err) - } else { + if !withForce { require.ErrorAs(t, err, &errLocked) + return + } + require.NoError(t, err) + + headPrm.WithAddress(object) + _, err = engine.Head(ctx, headPrm) + if withTombstone { + require.True(t, client.IsErrObjectAlreadyRemoved(err)) + } else { + require.True(t, client.IsErrObjectNotFound(err)) } } From 487cb34c5dcf21bf385d8ddf8a397652260101a8 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 21 Apr 2025 13:26:32 +0300 Subject: [PATCH 80/94] [#1689] engine/test: Refactor `TestInhumeIfObjectDoesntExist` test - Use the same storage engine in multiple parallel tests - Move `Lock`, `Inhume`, `Head` calls to separate functions Change-Id: I00849c1f068f0ab8d92061719d67d6fe786200db Signed-off-by: Aleksey Savchuk --- .../engine/inhume_test.go | 84 ++++++++++--------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index fa73dcad5..813c329f2 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -245,66 +245,74 @@ func benchmarkInhumeMultipart(b *testing.B, numShards, numObjects int) { } func TestInhumeIfObjectDoesntExist(t *testing.T) { + const numShards = 4 + + engine := testNewEngine(t).setShardsNum(t, numShards).prepare(t).engine + t.Cleanup(func() { require.NoError(t, engine.Close(context.Background())) }) + t.Run("object is locked", func(t *testing.T) { t.Run("inhume without tombstone", func(t *testing.T) { - testInhumeLockedIfObjectDoesntExist(t, false, false) + testInhumeLockedIfObjectDoesntExist(t, engine, false, false) }) t.Run("inhume with tombstone", func(t *testing.T) { - testInhumeLockedIfObjectDoesntExist(t, true, false) + testInhumeLockedIfObjectDoesntExist(t, engine, true, false) }) t.Run("force inhume", func(t *testing.T) { - testInhumeLockedIfObjectDoesntExist(t, false, true) + testInhumeLockedIfObjectDoesntExist(t, engine, false, true) }) }) } -func testInhumeLockedIfObjectDoesntExist(t *testing.T, withTombstone, withForce bool) { +func testInhumeLockedIfObjectDoesntExist(t *testing.T, e *StorageEngine, withTombstone, withForce bool) { t.Parallel() - // Due to the tests design it is possible to set both the options, - // however removal with tombstone and force removal are exclusive. - require.False(t, withTombstone && withForce) + object := oidtest.Address() + require.NoError(t, testLockObject(e, object)) - var ( - errLocked *apistatus.ObjectLocked - inhumePrm InhumePrm - headPrm HeadPrm - ctx = context.Background() - container = cidtest.ID() - object = oidtest.Address() - lock = oidtest.ID() - tombstone = oidtest.Address() - ) - object.SetContainer(container) - tombstone.SetContainer(container) - - engine := testNewEngine(t).setShardsNum(t, 4).prepare(t).engine - defer func() { require.NoError(t, engine.Close(ctx)) }() - - err := engine.Lock(ctx, container, lock, []oid.ID{object.Object()}) - require.NoError(t, err) - - if withTombstone { - inhumePrm.WithTarget(tombstone, object) - } else { - inhumePrm.MarkAsGarbage(object) - } - if withForce { - inhumePrm.WithForceRemoval() - } - - err = engine.Inhume(ctx, inhumePrm) + err := testInhumeObject(t, e, object, withTombstone, withForce) if !withForce { + var errLocked *apistatus.ObjectLocked require.ErrorAs(t, err, &errLocked) return } require.NoError(t, err) - headPrm.WithAddress(object) - _, err = engine.Head(ctx, headPrm) + err = testHeadObject(e, object) if withTombstone { require.True(t, client.IsErrObjectAlreadyRemoved(err)) } else { require.True(t, client.IsErrObjectNotFound(err)) } } + +func testLockObject(e *StorageEngine, obj oid.Address) error { + return e.Lock(context.Background(), obj.Container(), oidtest.ID(), []oid.ID{obj.Object()}) +} + +func testInhumeObject(t testing.TB, e *StorageEngine, obj oid.Address, withTombstone, withForce bool) error { + tombstone := oidtest.Address() + tombstone.SetContainer(obj.Container()) + + // Due to the tests design it is possible to set both the options, + // however removal with tombstone and force removal are exclusive. + require.False(t, withTombstone && withForce) + + var inhumePrm InhumePrm + if withTombstone { + inhumePrm.WithTarget(tombstone, obj) + } else { + inhumePrm.MarkAsGarbage(obj) + } + if withForce { + inhumePrm.WithForceRemoval() + } + return e.Inhume(context.Background(), inhumePrm) +} + +func testHeadObject(e *StorageEngine, obj oid.Address) error { + var headPrm HeadPrm + headPrm.WithAddress(obj) + + _, err := e.Head(context.Background(), headPrm) + return err +} From 77b8545601f13ba4e596542be58b7e6f9c8e36d1 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 21 Apr 2025 13:31:54 +0300 Subject: [PATCH 81/94] [#1689] engine: Fix removal of objects not found on node Ensured correct object status if the object is not found on a node. Fixed regression introduced in #1450. Besides an object not being found on any shard, it also important to remove it anyway in order to populate the metabase indexes because they are responsible for the correct object status, i.e., the status will be `object not found` without the indexes, the status will be `object is already removed` with the indexes. Change-Id: I6237fbc0f8bb0c4f2a51ada3a68f52950050e660 Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/inhume.go | 100 +++++++++++++++--- .../engine/inhume_test.go | 24 +++++ 2 files changed, 107 insertions(+), 17 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index e13f04927..e5f7072e2 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -74,7 +74,7 @@ func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) error { } func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) error { - addrsPerShard, err := e.groupObjectsByShard(ctx, prm.addrs, !prm.forceRemoval) + addrsPerShard, notFoundObjects, err := e.groupObjectsByShard(ctx, prm.addrs, !prm.forceRemoval) if err != nil { return err } @@ -84,8 +84,6 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) error { shPrm.ForceRemoval() } - var errLocked *apistatus.ObjectLocked - for shardID, addrs := range addrsPerShard { if prm.tombstone != nil { shPrm.SetTarget(*prm.tombstone, addrs...) @@ -103,39 +101,107 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) error { } if _, err := sh.Inhume(ctx, shPrm); err != nil { - switch { - case errors.As(err, &errLocked): - case errors.Is(err, shard.ErrLockObjectRemoval): - case errors.Is(err, shard.ErrReadOnlyMode): - case errors.Is(err, shard.ErrDegradedMode): - default: - e.reportShardError(ctx, sh, "couldn't inhume object in shard", err) - } + e.reportInhumeError(ctx, err, sh) return err } } - return nil + return e.inhumeNotFoundObjects(ctx, notFoundObjects, prm) +} + +func (e *StorageEngine) reportInhumeError(ctx context.Context, err error, hs hashedShard) { + if err == nil { + return + } + + var errLocked *apistatus.ObjectLocked + switch { + case errors.As(err, &errLocked): + case errors.Is(err, shard.ErrLockObjectRemoval): + case errors.Is(err, shard.ErrReadOnlyMode): + case errors.Is(err, shard.ErrDegradedMode): + default: + e.reportShardError(ctx, hs, "couldn't inhume object in shard", err) + } +} + +// inhumeNotFoundObjects removes object which are not found on any shard. +// +// Besides an object not being found on any shard, it is also important to +// remove it anyway in order to populate the metabase indexes because they are +// responsible for the correct object status, i.e., the status will be `object +// not found` without the indexes, the status will be `object is already +// removed` with the indexes. +// +// It is suggested to evenly remove those objects on each shard with the batch +// size equal to 1 + floor(number of objects / number of shards). +func (e *StorageEngine) inhumeNotFoundObjects(ctx context.Context, addrs []oid.Address, prm InhumePrm) error { + if len(addrs) == 0 { + return nil + } + + var shPrm shard.InhumePrm + if prm.forceRemoval { + shPrm.ForceRemoval() + } + + numObjectsPerShard := 1 + len(addrs)/len(e.shards) + + var inhumeErr error + itErr := e.iterateOverUnsortedShards(ctx, func(hs hashedShard) (stop bool) { + numObjects := min(numObjectsPerShard, len(addrs)) + + if numObjects == 0 { + return true + } + + if prm.tombstone != nil { + shPrm.SetTarget(*prm.tombstone, addrs[:numObjects]...) + } else { + shPrm.MarkAsGarbage(addrs[:numObjects]...) + } + addrs = addrs[numObjects:] + + _, inhumeErr = hs.Inhume(ctx, shPrm) + e.reportInhumeError(ctx, inhumeErr, hs) + return inhumeErr != nil + }) + if inhumeErr != nil { + return inhumeErr + } + return itErr } // groupObjectsByShard groups objects based on the shard(s) they are stored on. // // If checkLocked is set, [apistatus.ObjectLocked] will be returned if any of // the objects are locked. -func (e *StorageEngine) groupObjectsByShard(ctx context.Context, addrs []oid.Address, checkLocked bool) (map[string][]oid.Address, error) { - groups := make(map[string][]oid.Address) +// +// Returns two sets of objects: found objects which are grouped per shard and +// not found object. Not found objects are objects which are not found on any +// shard. This can happen if a node is a container node but doesn't participate +// in a replica group of the object. +func (e *StorageEngine) groupObjectsByShard(ctx context.Context, addrs []oid.Address, checkLocked bool) (groups map[string][]oid.Address, notFoundObjects []oid.Address, err error) { + groups = make(map[string][]oid.Address) + var ids []string for _, addr := range addrs { - ids, err := e.findShards(ctx, addr, checkLocked) + ids, err = e.findShards(ctx, addr, checkLocked) if err != nil { - return nil, err + return } + + if len(ids) == 0 { + notFoundObjects = append(notFoundObjects, addr) + continue + } + for _, id := range ids { groups[id] = append(groups[id], addr) } } - return groups, nil + return } // findShards determines the shard(s) where the object is stored. diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 813c329f2..0e268cd23 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -250,6 +250,16 @@ func TestInhumeIfObjectDoesntExist(t *testing.T) { engine := testNewEngine(t).setShardsNum(t, numShards).prepare(t).engine t.Cleanup(func() { require.NoError(t, engine.Close(context.Background())) }) + t.Run("inhume without tombstone", func(t *testing.T) { + testInhumeIfObjectDoesntExist(t, engine, false, false) + }) + t.Run("inhume with tombstone", func(t *testing.T) { + testInhumeIfObjectDoesntExist(t, engine, true, false) + }) + t.Run("force inhume", func(t *testing.T) { + testInhumeIfObjectDoesntExist(t, engine, false, true) + }) + t.Run("object is locked", func(t *testing.T) { t.Run("inhume without tombstone", func(t *testing.T) { testInhumeLockedIfObjectDoesntExist(t, engine, false, false) @@ -263,6 +273,20 @@ func TestInhumeIfObjectDoesntExist(t *testing.T) { }) } +func testInhumeIfObjectDoesntExist(t *testing.T, e *StorageEngine, withTombstone, withForce bool) { + t.Parallel() + + object := oidtest.Address() + require.NoError(t, testInhumeObject(t, e, object, withTombstone, withForce)) + + err := testHeadObject(e, object) + if withTombstone { + require.True(t, client.IsErrObjectAlreadyRemoved(err)) + } else { + require.True(t, client.IsErrObjectNotFound(err)) + } +} + func testInhumeLockedIfObjectDoesntExist(t *testing.T, e *StorageEngine, withTombstone, withForce bool) { t.Parallel() From 6bdbe6a18b2fc666af67837a3b5ce6434ddbea96 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 22 Apr 2025 17:37:29 +0300 Subject: [PATCH 82/94] [#1689] adm: Fix `printSubjectInfo` method * Fix print format for primary and additional subject keys from frostfsid contract. Since the format corresponds to `neo-go wallet dump-keys` output format. Change-Id: I9ae9fd43bfb378970786b97bd3d9d7f739466ae6 Signed-off-by: Airat Arifullin --- .../internal/modules/morph/frostfsid/frostfsid.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 8ae606f1a..7f777db98 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -1,6 +1,7 @@ package frostfsid import ( + "encoding/hex" "errors" "fmt" "math/big" @@ -604,7 +605,7 @@ func printSubjectInfo(cmd *cobra.Command, addr util.Uint160, subj *frostfsidclie cmd.Printf("Address: %s\n", address.Uint160ToString(addr)) pk := "" if subj.PrimaryKey != nil { - pk = subj.PrimaryKey.String() + pk = hex.EncodeToString(subj.PrimaryKey.Bytes()) } cmd.Printf("Primary key: %s\n", pk) cmd.Printf("Name: %s\n", subj.Name) @@ -614,7 +615,7 @@ func printSubjectInfo(cmd *cobra.Command, addr util.Uint160, subj *frostfsidclie for _, key := range subj.AdditionalKeys { k := "" if key != nil { - k = key.String() + k = hex.EncodeToString(key.Bytes()) } cmd.Printf("- %s\n", k) } From b0f39dca16c9564c6a68da874ceab705288dde9a Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 22 Apr 2025 18:14:00 +0300 Subject: [PATCH 83/94] [#1721] object: Make `CheckAPE` always validate bearer token * The bearer token must always be validated, regardless of whether it has been impersonated; * Fix unit-tests for tree service which check verification with bearer token. Close #1721 Change-Id: I5f715c498ae10b2e758244e60b8f21849328a04f Signed-off-by: Airat Arifullin --- pkg/services/common/ape/checker.go | 12 ++++++++---- pkg/services/tree/signature_test.go | 28 +++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index c9b0b7363..a2e628144 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -73,14 +73,18 @@ func New(localOverrideStorage policyengine.LocalOverrideStorage, morphChainStora // CheckAPE performs the common policy-engine check logic on a prepared request. func (c *checkerCoreImpl) CheckAPE(ctx context.Context, prm CheckPrm) error { var cr policyengine.ChainRouter - if prm.BearerToken != nil && !prm.BearerToken.Impersonate() { + if prm.BearerToken != nil { var err error if err = isValidBearer(prm.BearerToken, prm.ContainerOwner, prm.Container, prm.PublicKey, c.State); err != nil { return fmt.Errorf("bearer validation error: %w", err) } - cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, prm.BearerToken.APEOverride()) - if err != nil { - return fmt.Errorf("create chain router error: %w", err) + if prm.BearerToken.Impersonate() { + cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.MorphChainStorage, c.LocalOverrideStorage) + } else { + cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, prm.BearerToken.APEOverride()) + if err != nil { + return fmt.Errorf("create chain router error: %w", err) + } } } else { cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.MorphChainStorage, c.LocalOverrideStorage) diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index f5659d5e2..dd37b4191 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -238,14 +238,40 @@ func TestMessageSign(t *testing.T) { t.Run("impersonate", func(t *testing.T) { cnr.Value.SetBasicACL(acl.PublicRWExtended) var bt bearer.Token + bt.SetExp(10) bt.SetImpersonate(true) + bt.SetAPEOverride(bearer.APEOverride{ + Target: ape.ChainTarget{ + TargetType: ape.TargetTypeContainer, + Name: cid1.EncodeToString(), + }, + Chains: []ape.Chain{}, + }) + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + }) + + t.Run("impersonate but invalid signer", func(t *testing.T) { + var bt bearer.Token + bt.SetExp(10) + bt.SetImpersonate(true) + bt.SetAPEOverride(bearer.APEOverride{ + Target: ape.ChainTarget{ + TargetType: ape.TargetTypeContainer, + Name: cid1.EncodeToString(), + }, + Chains: []ape.Chain{}, + }) require.NoError(t, bt.Sign(privs[1].PrivateKey)) req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) From 5b6cba04cb5c3fb0589caa6e26bdc67c8ef183e2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Apr 2025 18:17:00 +0300 Subject: [PATCH 84/94] [#1689] node: Drop `node.relay` config It is not used and not tested properly, so drop it. Change-Id: I7c90c7391ecb4be17459415d209811ba1a693f7a Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/attributes.go | 4 --- cmd/frostfs-node/config.go | 27 +++++---------------- cmd/frostfs-node/config/node/config.go | 8 ------ cmd/frostfs-node/config/node/config_test.go | 5 ---- cmd/frostfs-node/netmap.go | 20 +++++---------- config/example/node.env | 1 - config/example/node.json | 1 - config/example/node.yaml | 2 -- docs/storage-node-configuration.md | 2 -- 9 files changed, 12 insertions(+), 58 deletions(-) diff --git a/cmd/frostfs-node/attributes.go b/cmd/frostfs-node/attributes.go index 64c3beba7..ce8ae9662 100644 --- a/cmd/frostfs-node/attributes.go +++ b/cmd/frostfs-node/attributes.go @@ -6,9 +6,5 @@ import ( ) func parseAttributes(c *cfg) { - if nodeconfig.Relay(c.appCfg) { - return - } - fatalOnErr(attributes.ReadNodeAttributes(&c.cfgNodeInfo.localInfo, nodeconfig.Attributes(c.appCfg))) } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index b688acfde..96274e625 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -651,7 +651,6 @@ type cfgNetmap struct { state *networkState - needBootstrap bool reBoostrapTurnedOff *atomic.Bool // managed by control service in runtime } @@ -710,11 +709,9 @@ func initCfg(appCfg *config.Config) *cfg { key := nodeconfig.Key(appCfg) - relayOnly := nodeconfig.Relay(appCfg) - netState := newNetworkState() - c.shared = initShared(appCfg, key, netState, relayOnly) + c.shared = initShared(appCfg, key, netState) netState.metrics = c.metricsCollector @@ -734,7 +731,7 @@ func initCfg(appCfg *config.Config) *cfg { c.cfgFrostfsID = initFrostfsID(appCfg) - c.cfgNetmap = initNetmap(appCfg, netState, relayOnly) + c.cfgNetmap = initNetmap(appCfg, netState) c.cfgGRPC = initCfgGRPC() @@ -780,12 +777,8 @@ func initSdNotify(appCfg *config.Config) bool { return false } -func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkState, relayOnly bool) shared { - var netAddr network.AddressGroup - - if !relayOnly { - netAddr = nodeconfig.BootstrapAddresses(appCfg) - } +func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkState) shared { + netAddr := nodeconfig.BootstrapAddresses(appCfg) persistate, err := state.NewPersistentStorage(nodeconfig.PersistentState(appCfg).Path()) fatalOnErr(err) @@ -836,18 +829,15 @@ func internalNetConfig(appCfg *config.Config, m metrics.MultinetMetrics) interna return result } -func initNetmap(appCfg *config.Config, netState *networkState, relayOnly bool) cfgNetmap { +func initNetmap(appCfg *config.Config, netState *networkState) cfgNetmap { netmapWorkerPool, err := ants.NewPool(notificationHandlerPoolSize) fatalOnErr(err) - var reBootstrapTurnedOff atomic.Bool - reBootstrapTurnedOff.Store(relayOnly) return cfgNetmap{ scriptHash: contractsconfig.Netmap(appCfg), state: netState, workerPool: netmapWorkerPool, - needBootstrap: !relayOnly, - reBoostrapTurnedOff: &reBootstrapTurnedOff, + reBoostrapTurnedOff: &atomic.Bool{}, } } @@ -1256,11 +1246,6 @@ func (c *cfg) bootstrap(ctx context.Context) error { return bootstrapOnline(ctx, c) } -// needBootstrap checks if local node should be registered in network on bootup. -func (c *cfg) needBootstrap() bool { - return c.cfgNetmap.needBootstrap -} - type dCmp struct { name string reloadFunc func() error diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index 81b191e96..c50718c5f 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -131,14 +131,6 @@ func Attributes(c *config.Config) (attrs []string) { return } -// Relay returns the value of "relay" config parameter -// from "node" section. -// -// Returns false if the value is not set. -func Relay(c *config.Config) bool { - return config.BoolSafe(c.Sub(subsection), "relay") -} - // PersistentSessions returns structure that provides access to "persistent_sessions" // subsection of "node" section. func PersistentSessions(c *config.Config) PersistentSessionsConfig { diff --git a/cmd/frostfs-node/config/node/config_test.go b/cmd/frostfs-node/config/node/config_test.go index 7b9adecf4..9af1dc038 100644 --- a/cmd/frostfs-node/config/node/config_test.go +++ b/cmd/frostfs-node/config/node/config_test.go @@ -29,12 +29,10 @@ func TestNodeSection(t *testing.T) { ) attribute := Attributes(empty) - relay := Relay(empty) persisessionsPath := PersistentSessions(empty).Path() persistatePath := PersistentState(empty).Path() require.Empty(t, attribute) - require.Equal(t, false, relay) require.Equal(t, "", persisessionsPath) require.Equal(t, PersistentStatePathDefault, persistatePath) }) @@ -45,7 +43,6 @@ func TestNodeSection(t *testing.T) { key := Key(c) addrs := BootstrapAddresses(c) attributes := Attributes(c) - relay := Relay(c) wKey := Wallet(c) persisessionsPath := PersistentSessions(c).Path() persistatePath := PersistentState(c).Path() @@ -87,8 +84,6 @@ func TestNodeSection(t *testing.T) { return false }) - require.Equal(t, true, relay) - require.Len(t, attributes, 2) require.Equal(t, "Price:11", attributes[0]) require.Equal(t, "UN-LOCODE:RU MSK", attributes[1]) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 82b799e4c..7dfb4fe12 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -187,7 +187,7 @@ func addNewEpochNotificationHandlers(c *cfg) { c.updateContractNodeInfo(ctx, e) - if !c.needBootstrap() || c.cfgNetmap.reBoostrapTurnedOff.Load() { // fixes #470 + if c.cfgNetmap.reBoostrapTurnedOff.Load() { // fixes #470 return } @@ -209,14 +209,12 @@ func addNewEpochNotificationHandlers(c *cfg) { // bootstrapNode adds current node to the Network map. // Must be called after initNetmapService. func bootstrapNode(ctx context.Context, c *cfg) { - if c.needBootstrap() { - if c.IsMaintenance() { - c.log.Info(ctx, logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) - return - } - err := c.bootstrap(ctx) - fatalOnErrDetails("bootstrap error", err) + if c.IsMaintenance() { + c.log.Info(ctx, logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) + return } + err := c.bootstrap(ctx) + fatalOnErrDetails("bootstrap error", err) } func addNetmapNotificationHandler(c *cfg, sTyp string, h event.Handler) { @@ -352,8 +350,6 @@ func addNewEpochAsyncNotificationHandler(c *cfg, h event.Handler) { ) } -var errRelayBootstrap = errors.New("setting netmap status is forbidden in relay mode") - func (c *cfg) SetNetmapStatus(ctx context.Context, st control.NetmapStatus) error { switch st { default: @@ -365,10 +361,6 @@ func (c *cfg) SetNetmapStatus(ctx context.Context, st control.NetmapStatus) erro c.stopMaintenance(ctx) - if !c.needBootstrap() { - return errRelayBootstrap - } - if st == control.NetmapStatus_ONLINE { c.cfgNetmap.reBoostrapTurnedOff.Store(false) return bootstrapOnline(ctx, c) diff --git a/config/example/node.env b/config/example/node.env index e7d7a6cc8..9a2426358 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -22,7 +22,6 @@ FROSTFS_NODE_WALLET_PASSWORD=password FROSTFS_NODE_ADDRESSES="s01.frostfs.devenv:8080 /dns4/s02.frostfs.devenv/tcp/8081 grpc://127.0.0.1:8082 grpcs://localhost:8083" FROSTFS_NODE_ATTRIBUTE_0=Price:11 FROSTFS_NODE_ATTRIBUTE_1="UN-LOCODE:RU MSK" -FROSTFS_NODE_RELAY=true FROSTFS_NODE_PERSISTENT_SESSIONS_PATH=/sessions FROSTFS_NODE_PERSISTENT_STATE_PATH=/state FROSTFS_NODE_LOCODE_DB_PATH=/path/to/locode/db diff --git a/config/example/node.json b/config/example/node.json index 3f7854d98..6b7a9c2c6 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -37,7 +37,6 @@ ], "attribute_0": "Price:11", "attribute_1": "UN-LOCODE:RU MSK", - "relay": true, "persistent_sessions": { "path": "/sessions" }, diff --git a/config/example/node.yaml b/config/example/node.yaml index 32f0cba67..2d4bc90fb 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -34,7 +34,6 @@ node: - grpcs://localhost:8083 attribute_0: "Price:11" attribute_1: UN-LOCODE:RU MSK - relay: true # start Storage node in relay mode without bootstrapping into the Network map persistent_sessions: path: /sessions # path to persistent session tokens file of Storage node (default: in-memory sessions) persistent_state: @@ -141,7 +140,6 @@ rpc: max_ops: 10000 storage: - # note: shard configuration can be omitted for relay node (see `node.relay`) shard_ro_error_threshold: 100 # amount of errors to occur before shard is made read-only (default: 0, ignore errors) shard: diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 14ebb53b3..da9fdfed0 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -410,7 +410,6 @@ node: - "Price:11" - "UN-LOCODE:RU MSK" - "key:value" - relay: false persistent_sessions: path: /sessions persistent_state: @@ -424,7 +423,6 @@ node: | `wallet` | [Wallet config](#wallet-subsection) | | Wallet configuration. Has no effect if `key` is provided. | | `addresses` | `[]string` | | Addresses advertised in the netmap. | | `attribute` | `[]string` | | Node attributes as a list of key-value pairs in `:` format. | -| `relay` | `bool` | | Enable relay mode. | | `persistent_sessions` | [Persistent sessions config](#persistent_sessions-subsection) | | Persistent session token store configuration. | | `persistent_state` | [Persistent state config](#persistent_state-subsection) | | Persistent state configuration. | | `locode_db_path` | `string` | empty | Path to UN/LOCODE [database](https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/) for FrostFS. | From 652237500fb9e1618eb66310d3f2a51c599a033e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 24 Apr 2025 16:54:47 +0300 Subject: [PATCH 85/94] [#1689] tree: Dial tree service after create connection on sync Background trees sync creates grpc connection with `grpc.WithDefaultCallOptions(grpc.WaitForReady(true))` option. When grpc connection created with this option, client will wait until a connection becomes available or the RPC's deadline is reached. As background sync has no timeout in context, so in case of client is in TRANSIENT_FAILURE RPC call will hang forever. Change-Id: I17c8c1d2779bb81c541f47dd0e558e0b8ed2e7c1 Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/cache.go | 36 ++++++------------------------------ pkg/services/tree/sync.go | 28 +++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 462c8554f..a11700771 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -66,7 +66,12 @@ func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceCl } } - cc, err := c.dialTreeService(ctx, netmapAddr) + var netAddr network.Address + if err := netAddr.FromString(netmapAddr); err != nil { + return nil, err + } + + cc, err := dialTreeService(ctx, netAddr, c.key, c.ds) lastTry := time.Now() c.Lock() @@ -83,32 +88,3 @@ func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceCl return NewTreeServiceClient(cc), nil } - -func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, error) { - var netAddr network.Address - if err := netAddr.FromString(netmapAddr); err != nil { - return nil, err - } - - cc, err := createConnection(netAddr, grpc.WithContextDialer(c.ds.GrpcContextDialer())) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithTimeout(ctx, defaultClientConnectTimeout) - defer cancel() - - req := &HealthcheckRequest{ - Body: &HealthcheckRequest_Body{}, - } - if err := SignMessage(req, c.key); err != nil { - return nil, err - } - - // perform some request to check connection - if _, err := NewTreeServiceClient(cc).Healthcheck(ctx, req); err != nil { - _ = cc.Close() - return nil, err - } - return cc, nil -} diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index d4040337d..af355639f 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -2,6 +2,7 @@ package tree import ( "context" + "crypto/ecdsa" "crypto/sha256" "crypto/tls" "errors" @@ -14,6 +15,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -304,7 +306,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, continue } - cc, err := createConnection(a, grpc.WithContextDialer(s.ds.GrpcContextDialer())) + cc, err := dialTreeService(ctx, a, s.key, s.ds) if err != nil { s.log.Warn(ctx, logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr)) continue @@ -342,6 +344,30 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return from } +func dialTreeService(ctx context.Context, netAddr network.Address, key *ecdsa.PrivateKey, ds *net.DialerSource) (*grpc.ClientConn, error) { + cc, err := createConnection(netAddr, grpc.WithContextDialer(ds.GrpcContextDialer())) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(ctx, defaultClientConnectTimeout) + defer cancel() + + req := &HealthcheckRequest{ + Body: &HealthcheckRequest_Body{}, + } + if err := SignMessage(req, key); err != nil { + return nil, err + } + + // perform some request to check connection + if _, err := NewTreeServiceClient(cc).Healthcheck(ctx, req); err != nil { + _ = cc.Close() + return nil, err + } + return cc, nil +} + func createConnection(a network.Address, opts ...grpc.DialOption) (*grpc.ClientConn, error) { host, isTLS, err := client.ParseURI(a.URIAddr()) if err != nil { From 8e2f919df0fe4525bf25fcbe1fa11f6af642637e Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 28 Apr 2025 20:07:56 +0300 Subject: [PATCH 86/94] [#1689] go.mod: Bump SDK version * Fix `APEOverride` method usage in ape checker. * Fix linter errors: factor out deprecated methods and packages. Change-Id: I8c939f4c58c2a4e3c4e795c7224d935d40ce6f24 Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/container/get.go | 4 +- cmd/frostfs-cli/modules/container/list.go | 4 +- go.mod | 24 ++++----- go.sum | 52 +++++++++---------- pkg/core/container/util.go | 4 +- .../internal/testutil/generators.go | 7 +-- .../internal/testutil/object.go | 2 +- pkg/services/common/ape/checker.go | 5 +- .../populate-metabase/internal/generate.go | 5 +- 9 files changed, 55 insertions(+), 52 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/get.go b/cmd/frostfs-cli/modules/container/get.go index 8c4ab14f8..fac6eb2cd 100644 --- a/cmd/frostfs-cli/modules/container/get.go +++ b/cmd/frostfs-cli/modules/container/get.go @@ -93,9 +93,9 @@ func prettyPrintContainer(cmd *cobra.Command, cnr container.Container, jsonEncod cmd.Println("created:", container.CreatedAt(cnr)) cmd.Println("attributes:") - cnr.IterateAttributes(func(key, val string) { + for key, val := range cnr.Attributes() { cmd.Printf("\t%s=%s\n", key, val) - }) + } cmd.Println("placement policy:") commonCmd.ExitOnErr(cmd, "write policy: %w", cnr.PlacementPolicy().WriteStringTo((*stringWriter)(cmd))) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index bbb8da840..e4a023d91 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -102,9 +102,9 @@ func printContainer(cmd *cobra.Command, prmGet internalclient.GetContainerPrm, i cmd.Println(id.String()) if flagVarListPrintAttr { - cnr.IterateUserAttributes(func(key, val string) { + for key, val := range cnr.Attributes() { cmd.Printf(" %s: %s\n", key, val) - }) + } } } diff --git a/go.mod b/go.mod index 5ed4a90be..fb45c3874 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.23 +go 1.23.0 require ( code.gitea.io/sdk/gitea v0.17.1 @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.5.2 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250428134706-8822aedbbbaa git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88 @@ -28,7 +28,7 @@ require ( github.com/klauspost/compress v1.17.4 github.com/mailru/easyjson v0.7.7 github.com/mr-tron/base58 v1.2.0 - github.com/multiformats/go-multiaddr v0.14.0 + github.com/multiformats/go-multiaddr v0.15.0 github.com/nspcc-dev/neo-go v0.106.3 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 @@ -44,10 +44,9 @@ require ( go.opentelemetry.io/otel v1.31.0 go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/sync v0.10.0 - golang.org/x/sys v0.28.0 - golang.org/x/term v0.27.0 + golang.org/x/sync v0.12.0 + golang.org/x/sys v0.31.0 + golang.org/x/term v0.30.0 google.golang.org/grpc v1.69.2 google.golang.org/protobuf v1.36.1 gopkg.in/yaml.v3 v3.0.1 @@ -86,9 +85,9 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/ipfs/go-cid v0.4.1 // indirect + github.com/ipfs/go-cid v0.5.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/klauspost/reedsolomon v1.12.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -124,13 +123,14 @@ require ( go.opentelemetry.io/otel/sdk v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect golang.org/x/net v0.30.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/text v0.23.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - lukechampine.com/blake3 v1.2.1 // indirect + lukechampine.com/blake3 v1.4.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 38aba9bde..acc26af36 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47 h1:O2c3VOlaGZ862hf2ZPLBMdTG6vGJzhIgDvFEFGfntzU= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d h1:ZLKDupw362Ciing7kdIZhDYGMyo2QZyJ6sS/8X9QWJ0= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d/go.mod h1:2PWt5GwJTnhjHp+mankcfCeAJBMn7puxPm+RS+lliVk= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250428134706-8822aedbbbaa h1:ttJxiw5+Wti3outhaPFaLGwCinmUTQgyVQfD/sIU5sg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250428134706-8822aedbbbaa/go.mod h1:mimnb6yQUBLLQ8PboNc5ZP8iz4VMhFRKrfZcjfR9CVs= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -145,14 +145,14 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= -github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= +github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/reedsolomon v1.12.1 h1:NhWgum1efX1x58daOBGCFWcxtEhOhXKKl1HAPQUp03Q= github.com/klauspost/reedsolomon v1.12.1/go.mod h1:nEi5Kjb6QqtbofI6s+cbG/j1da11c96IBYBSnVGtuBs= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -192,8 +192,8 @@ github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aG github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU= -github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4= +github.com/multiformats/go-multiaddr v0.15.0 h1:zB/HeaI/apcZiTDwhY5YqMvNVl/oQYvs3XySU+qeAVo= +github.com/multiformats/go-multiaddr v0.15.0/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= @@ -324,15 +324,15 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -353,8 +353,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -381,16 +381,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -398,15 +398,15 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -445,7 +445,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= -lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +lukechampine.com/blake3 v1.4.0 h1:xDbKOZCVbnZsfzM6mHSYcGRHZ3YrLDzqz8XnV4uaD5w= +lukechampine.com/blake3 v1.4.0/go.mod h1:MQJNQCTnR+kwOP/JEZSxj3MaQjp80FOFSNMMHXcSeX0= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/pkg/core/container/util.go b/pkg/core/container/util.go index a24b36944..61c568052 100644 --- a/pkg/core/container/util.go +++ b/pkg/core/container/util.go @@ -26,10 +26,10 @@ func WasRemoved(ctx context.Context, s Source, cid cid.ID) (bool, error) { // IsIndexedContainer returns True if container attributes should be indexed. func IsIndexedContainer(cnr containerSDK.Container) bool { var isS3Container bool - cnr.IterateAttributes(func(key, _ string) { + for key := range cnr.Attributes() { if key == ".s3-location-constraint" { isS3Container = true } - }) + } return !isS3Container } diff --git a/pkg/local_object_storage/internal/testutil/generators.go b/pkg/local_object_storage/internal/testutil/generators.go index 383c596af..52b199b0b 100644 --- a/pkg/local_object_storage/internal/testutil/generators.go +++ b/pkg/local_object_storage/internal/testutil/generators.go @@ -1,7 +1,9 @@ package testutil import ( + cryptorand "crypto/rand" "encoding/binary" + "math/rand" "sync/atomic" "testing" @@ -9,7 +11,6 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" - "golang.org/x/exp/rand" ) // AddressGenerator is the interface of types that generate object addresses. @@ -61,7 +62,7 @@ var _ ObjectGenerator = &SeqObjGenerator{} func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *objectSDK.Object { data := make([]byte, sz) - _, _ = rand.Read(data) + _, _ = cryptorand.Read(data) obj := GenerateObjectWithCIDWithPayload(cid, data) obj.SetID(oid) return obj @@ -82,7 +83,7 @@ var _ ObjectGenerator = &RandObjGenerator{} func (g *RandObjGenerator) Next() *objectSDK.Object { var id oid.ID - _, _ = rand.Read(id[:]) + _, _ = cryptorand.Read(id[:]) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) } diff --git a/pkg/local_object_storage/internal/testutil/object.go b/pkg/local_object_storage/internal/testutil/object.go index 60e9211d5..1087e40be 100644 --- a/pkg/local_object_storage/internal/testutil/object.go +++ b/pkg/local_object_storage/internal/testutil/object.go @@ -1,6 +1,7 @@ package testutil import ( + "crypto/rand" "crypto/sha256" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" @@ -11,7 +12,6 @@ import ( usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/tzhash/tz" - "golang.org/x/exp/rand" ) const defaultDataSize = 32 diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index a2e628144..02c1e2ee1 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -81,7 +81,8 @@ func (c *checkerCoreImpl) CheckAPE(ctx context.Context, prm CheckPrm) error { if prm.BearerToken.Impersonate() { cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.MorphChainStorage, c.LocalOverrideStorage) } else { - cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, prm.BearerToken.APEOverride()) + override, _ := prm.BearerToken.APEOverride() + cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, override) if err != nil { return fmt.Errorf("create chain router error: %w", err) } @@ -130,7 +131,7 @@ func isValidBearer(token *bearer.Token, ownerCnr user.ID, cntID cid.ID, publicKe } // Check for ape overrides defined in the bearer token. - apeOverride := token.APEOverride() + apeOverride, _ := token.APEOverride() if len(apeOverride.Chains) > 0 && apeOverride.Target.TargetType != ape.TargetTypeContainer { return fmt.Errorf("%w: %s", errInvalidTargetType, apeOverride.Target.TargetType.ToV2().String()) } diff --git a/scripts/populate-metabase/internal/generate.go b/scripts/populate-metabase/internal/generate.go index f2f8881cf..39a420358 100644 --- a/scripts/populate-metabase/internal/generate.go +++ b/scripts/populate-metabase/internal/generate.go @@ -1,8 +1,10 @@ package internal import ( + cryptorand "crypto/rand" "crypto/sha256" "fmt" + "math/rand" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -14,14 +16,13 @@ import ( usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/tzhash/tz" - "golang.org/x/exp/rand" ) func GeneratePayloadPool(count uint, size uint) [][]byte { var pool [][]byte for range count { payload := make([]byte, size) - _, _ = rand.Read(payload) + _, _ = cryptorand.Read(payload) pool = append(pool, payload) } From 64b46746e409a40a93a41c030a646d69fbf45282 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 28 Apr 2025 20:10:09 +0300 Subject: [PATCH 87/94] [#1689] ape: Fix validation for overrides in bearer * APE-overrides are optional for bearer. So, it should validate only set override; * Bearer can set overrides for containers, not only the one container - validation expects for any target type for set override. Basically, APE-overrides for all container must be set for namespace target; * Add unit-test cases to check bearer token validation. Change-Id: I6b8e19eb73d24f8cd8799bf99b6c551287da67d9 Signed-off-by: Airat Arifullin --- pkg/services/common/ape/checker.go | 32 ++++++++------- pkg/services/tree/signature_test.go | 61 +++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index 02c1e2ee1..fcd3efa44 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -20,7 +20,6 @@ import ( ) var ( - errInvalidTargetType = errors.New("bearer token defines non-container target override") errBearerExpired = errors.New("bearer token has expired") errBearerInvalidSignature = errors.New("bearer token has invalid signature") errBearerInvalidContainerID = errors.New("bearer token was created for another container") @@ -81,7 +80,10 @@ func (c *checkerCoreImpl) CheckAPE(ctx context.Context, prm CheckPrm) error { if prm.BearerToken.Impersonate() { cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.MorphChainStorage, c.LocalOverrideStorage) } else { - override, _ := prm.BearerToken.APEOverride() + override, isSet := prm.BearerToken.APEOverride() + if !isSet { + return errors.New("expected for override within bearer") + } cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, override) if err != nil { return fmt.Errorf("create chain router error: %w", err) @@ -131,19 +133,19 @@ func isValidBearer(token *bearer.Token, ownerCnr user.ID, cntID cid.ID, publicKe } // Check for ape overrides defined in the bearer token. - apeOverride, _ := token.APEOverride() - if len(apeOverride.Chains) > 0 && apeOverride.Target.TargetType != ape.TargetTypeContainer { - return fmt.Errorf("%w: %s", errInvalidTargetType, apeOverride.Target.TargetType.ToV2().String()) - } - - // Then check if container is either empty or equal to the container in the request. - var targetCnr cid.ID - err := targetCnr.DecodeString(apeOverride.Target.Name) - if err != nil { - return fmt.Errorf("invalid cid format: %s", apeOverride.Target.Name) - } - if !cntID.Equals(targetCnr) { - return errBearerInvalidContainerID + if apeOverride, isSet := token.APEOverride(); isSet { + switch apeOverride.Target.TargetType { + case ape.TargetTypeContainer: + var targetCnr cid.ID + err := targetCnr.DecodeString(apeOverride.Target.Name) + if err != nil { + return fmt.Errorf("invalid cid format: %s", apeOverride.Target.Name) + } + if !cntID.Equals(targetCnr) { + return errBearerInvalidContainerID + } + default: + } } // Then check if container owner signed this token. diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index dd37b4191..13a5c1395 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -235,6 +235,48 @@ func TestMessageSign(t *testing.T) { require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) + t.Run("omit override within bt", func(t *testing.T) { + t.Run("personated", func(t *testing.T) { + bt := testBearerTokenNoOverride() + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + + require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) + require.ErrorContains(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut), "expected for override") + }) + + t.Run("impersonated", func(t *testing.T) { + bt := testBearerTokenNoOverride() + bt.SetImpersonate(true) + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + + require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + }) + }) + + t.Run("invalid override within bearer token", func(t *testing.T) { + t.Run("personated", func(t *testing.T) { + bt := testBearerTokenCorruptOverride(privs[1].PublicKey(), privs[2].PublicKey()) + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + + require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) + require.ErrorContains(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut), "invalid cid") + }) + + t.Run("impersonated", func(t *testing.T) { + bt := testBearerTokenCorruptOverride(privs[1].PublicKey(), privs[2].PublicKey()) + bt.SetImpersonate(true) + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + + require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) + require.ErrorContains(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut), "invalid cid") + }) + }) + t.Run("impersonate", func(t *testing.T) { cnr.Value.SetBasicACL(acl.PublicRWExtended) var bt bearer.Token @@ -311,6 +353,25 @@ func testBearerToken(cid cid.ID, forPutGet, forGet *keys.PublicKey) bearer.Token return b } +func testBearerTokenCorruptOverride(forPutGet, forGet *keys.PublicKey) bearer.Token { + var b bearer.Token + b.SetExp(currentEpoch + 1) + b.SetAPEOverride(bearer.APEOverride{ + Target: ape.ChainTarget{ + TargetType: ape.TargetTypeContainer, + }, + Chains: []ape.Chain{{Raw: testChain(forPutGet, forGet).Bytes()}}, + }) + + return b +} + +func testBearerTokenNoOverride() bearer.Token { + var b bearer.Token + b.SetExp(currentEpoch + 1) + return b +} + func testChain(forPutGet, forGet *keys.PublicKey) *chain.Chain { ruleGet := chain.Rule{ Status: chain.Allow, From e0a4835ea3eb1e2dfda921319c86f4de69b686fe Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 24 Apr 2025 18:48:24 +0300 Subject: [PATCH 88/94] [#1689] go.mod: Bump `policy-engine` package version * The bumped version of `policy-engine` package introduces new native schema properties for resource. Change-Id: If392c51415cff66c46798a788a2f0944f15900d3 Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fb45c3874..6f1950936 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250428134706-8822aedbbbaa git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250425083815-09ff3bf14991 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/VictoriaMetrics/easyproto v0.1.4 diff --git a/go.sum b/go.sum index acc26af36..5b075f60a 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/96 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972/go.mod h1:2hM42MBrlhvN6XToaW6OWNk5ZLcu1FhaukGgxtfpDDI= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07 h1:gPaqGsk6gSWQyNVjaStydfUz6Z/loHc9XyvGrJ5qSPY= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88 h1:V0a7ia84ZpSM2YxpJq1SKLQfeYmsqFWqcxwweBHJIzc= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88/go.mod h1:GZTk55RI4dKzsK6BCn5h2xxE28UHNfgoq/NJxW/LQ6A= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250425083815-09ff3bf14991 h1:eTefR8y2y9cg7X5kybIcXDdmABfk/3A2awdmFD3zOsA= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250425083815-09ff3bf14991/go.mod h1:GZTk55RI4dKzsK6BCn5h2xxE28UHNfgoq/NJxW/LQ6A= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= From 6cedfbc17a871f6d96110b63071760488dbccb19 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 May 2025 16:20:13 +0300 Subject: [PATCH 89/94] [#1689] getSvc: Stop node addresses iteratation on logical errors Before this fix: If first node address returns non-logical error (context deadline exceeded, context canceled, etc), then this error will override logical error from second node address. Change-Id: Ib30d0209828fdc83b55308ca2e33a361aa4caee6 Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/v2/util.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 4b7dcc530..e699a3779 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -3,6 +3,7 @@ package getsvc import ( "context" "crypto/sha256" + "errors" "hash" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" @@ -358,19 +359,20 @@ func groupAddressRequestForwarder(f func(context.Context, network.Address, clien info.AddressGroup().IterateAddresses(func(addr network.Address) (stop bool) { var err error - - defer func() { - stop = err == nil - - if stop || firstErr == nil { - firstErr = err - } - - // would be nice to log otherwise - }() - res, err = f(ctx, addr, c, key) + // non-status logic error that could be returned + // from the SDK client; should not be considered + // as a connection error + var siErr *objectSDK.SplitInfoError + var eiErr *objectSDK.ECInfoError + + stop = err == nil || errors.As(err, &siErr) || errors.As(err, &eiErr) + + if stop || firstErr == nil { + firstErr = err + } + return }) From a5f76a609deb8a414685015fb7a39d5c99ad085c Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 5 May 2025 16:33:07 +0300 Subject: [PATCH 90/94] [#1689] ape: Fix bearer token validation * Request's sender is set to the token's issuer's public key if it's impersonated. Thus, token's user assertion must be fixed; * Add unit-test: check impersonated token but set user with `ForUser`. Change-Id: I5e299947761e237b1b4b339cf2d1278ef518239d Signed-off-by: Airat Arifullin --- pkg/services/common/ape/checker.go | 12 ++++++++++-- pkg/services/tree/signature_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index fcd3efa44..eb6263320 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -157,8 +157,16 @@ func isValidBearer(token *bearer.Token, ownerCnr user.ID, cntID cid.ID, publicKe var usrSender user.ID user.IDFromKey(&usrSender, (ecdsa.PublicKey)(*publicKey)) - if !token.AssertUser(usrSender) { - return errBearerInvalidOwner + // Then check if sender is valid. If it is an impersonated token, the sender is set to the token's issuer's + // public key, but not the actual sender. + if !token.Impersonate() { + if !token.AssertUser(usrSender) { + return errBearerInvalidOwner + } + } else { + if !bearer.ResolveIssuer(*token).Equals(usrSender) { + return errBearerInvalidOwner + } } return nil diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 13a5c1395..8815c227f 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -297,6 +297,30 @@ func TestMessageSign(t *testing.T) { require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) + t.Run("impersonate, but target user is still set", func(t *testing.T) { + var bt bearer.Token + bt.SetExp(10) + bt.SetImpersonate(true) + + var reqSigner user.ID + user.IDFromKey(&reqSigner, (ecdsa.PublicKey)(*privs[1].PublicKey())) + + bt.ForUser(reqSigner) + bt.SetAPEOverride(bearer.APEOverride{ + Target: ape.ChainTarget{ + TargetType: ape.TargetTypeContainer, + Name: cid1.EncodeToString(), + }, + Chains: []ape.Chain{}, + }) + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + + require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + }) + t.Run("impersonate but invalid signer", func(t *testing.T) { var bt bearer.Token bt.SetExp(10) From c2a495814f32b9ccfe3efedff76937ba03a61858 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 24 Apr 2025 18:57:23 +0300 Subject: [PATCH 91/94] [#1689] object: Make APE middleware form container system attributes * Extract container system attributes into request info; * Form APE-resource proprties from the extracted attributes; * Fix unit-test. Change-Id: I8fbd9a167ad05af0e75df350ac882b866da7bdcb Signed-off-by: Airat Arifullin --- pkg/services/object/ape/checker.go | 3 + pkg/services/object/ape/metadata.go | 7 + pkg/services/object/ape/request.go | 9 +- pkg/services/object/ape/request_test.go | 20 ++- pkg/services/object/ape/service.go | 177 +++++++++++++----------- 5 files changed, 128 insertions(+), 88 deletions(-) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index b96757def..bb6067a37 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -64,6 +64,9 @@ type Prm struct { // An encoded container's owner user ID. ContainerOwner user.ID + // Attributes defined for the container. + ContainerAttributes map[string]string + // The request's bearer token. It is used in order to check APE overrides with the token. BearerToken *bearer.Token diff --git a/pkg/services/object/ape/metadata.go b/pkg/services/object/ape/metadata.go index b37c3b6f8..102985aa6 100644 --- a/pkg/services/object/ape/metadata.go +++ b/pkg/services/object/ape/metadata.go @@ -63,6 +63,8 @@ type RequestInfo struct { ContainerOwner user.ID + ContainerAttributes map[string]string + // Namespace defines to which namespace a container is belonged. Namespace string @@ -131,6 +133,11 @@ func (e *extractor) GetRequestInfo(ctx context.Context, m Metadata, method strin ri.Role = nativeSchemaRole(res.Role) ri.ContainerOwner = cnr.Value.Owner() + ri.ContainerAttributes = map[string]string{} + for key, val := range cnr.Value.Attributes() { + ri.ContainerAttributes[key] = val + } + cnrNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(cnr.Value).Zone(), ".ns") if hasNamespace { ri.Namespace = cnrNamespace diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index 001a5f71e..39dd7f476 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -57,11 +57,16 @@ func resourceName(cid cid.ID, oid *oid.ID, namespace string) string { } // objectProperties collects object properties from address parameters and a header if it is passed. -func objectProperties(cnr cid.ID, oid *oid.ID, cnrOwner user.ID, header *objectV2.Header) map[string]string { +func objectProperties(cnr cid.ID, oid *oid.ID, cnrOwner user.ID, cnrAttrs map[string]string, header *objectV2.Header) map[string]string { objectProps := map[string]string{ nativeschema.PropertyKeyObjectContainerID: cnr.EncodeToString(), } + for attrName, attrValue := range cnrAttrs { + prop := fmt.Sprintf(nativeschema.PropertyKeyFormatObjectContainerAttribute, attrName) + objectProps[prop] = attrValue + } + objectProps[nativeschema.PropertyKeyContainerOwnerID] = cnrOwner.EncodeToString() if oid != nil { @@ -155,7 +160,7 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re prm.Method, aperequest.NewResource( resourceName(prm.Container, prm.Object, prm.Namespace), - objectProperties(prm.Container, prm.Object, prm.ContainerOwner, header), + objectProperties(prm.Container, prm.Object, prm.ContainerOwner, prm.ContainerAttributes, header), ), reqProps, ), nil diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go index f270bf97d..fcf7c4c40 100644 --- a/pkg/services/object/ape/request_test.go +++ b/pkg/services/object/ape/request_test.go @@ -7,6 +7,7 @@ import ( "testing" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" + cnrV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -22,8 +23,17 @@ const ( testOwnerID = "NURFM8PWbLA2aLt2vrD8q4FyfAdgESwM8y" incomingIP = "192.92.33.1" + + testSysAttrName = "unittest" + + testSysAttrZone = "eggplant" ) +var containerAttrs = map[string]string{ + cnrV2.SysAttributeName: testSysAttrName, + cnrV2.SysAttributeZone: testSysAttrZone, +} + func ctxWithPeerInfo() context.Context { return peer.NewContext(context.Background(), &peer.Peer{ Addr: &net.TCPAddr{ @@ -105,7 +115,7 @@ func TestObjectProperties(t *testing.T) { var testCnrOwner user.ID require.NoError(t, testCnrOwner.DecodeString(testOwnerID)) - props := objectProperties(cnr, obj, testCnrOwner, header.ToV2().GetHeader()) + props := objectProperties(cnr, obj, testCnrOwner, containerAttrs, header.ToV2().GetHeader()) require.Equal(t, test.container, props[nativeschema.PropertyKeyObjectContainerID]) require.Equal(t, testOwnerID, props[nativeschema.PropertyKeyContainerOwnerID]) @@ -124,6 +134,8 @@ func TestObjectProperties(t *testing.T) { require.Equal(t, test.header.typ.String(), props[nativeschema.PropertyKeyObjectType]) require.Equal(t, test.header.payloadChecksum.String(), props[nativeschema.PropertyKeyObjectPayloadHash]) require.Equal(t, test.header.payloadHomomorphicHash.String(), props[nativeschema.PropertyKeyObjectHomomorphicHash]) + require.Equal(t, containerAttrs[cnrV2.SysAttributeName], props[fmt.Sprintf(nativeschema.PropertyKeyFormatObjectContainerAttribute, cnrV2.SysAttributeName)]) + require.Equal(t, containerAttrs[cnrV2.SysAttributeZone], props[fmt.Sprintf(nativeschema.PropertyKeyFormatObjectContainerAttribute, cnrV2.SysAttributeZone)]) for _, attr := range test.header.attributes { require.Equal(t, attr.val, props[attr.key]) @@ -245,6 +257,10 @@ func TestNewAPERequest(t *testing.T) { Role: role, SenderKey: senderKey, ContainerOwner: testCnrOwner, + ContainerAttributes: map[string]string{ + cnrV2.SysAttributeZone: testSysAttrZone, + cnrV2.SysAttributeName: testSysAttrName, + }, } headerSource := newHeaderProviderMock() @@ -277,7 +293,7 @@ func TestNewAPERequest(t *testing.T) { method, aperequest.NewResource( resourceName(cnr, obj, prm.Namespace), - objectProperties(cnr, obj, testCnrOwner, func() *objectV2.Header { + objectProperties(cnr, obj, testCnrOwner, containerAttrs, func() *objectV2.Header { if headerObjSDK != nil { return headerObjSDK.ToV2().GetHeader() } diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index e199e2638..5e04843f3 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -86,16 +86,17 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { } prm := Prm{ - Namespace: g.reqInfo.Namespace, - Container: cnrID, - Object: objID, - Header: partInit.GetHeader(), - Method: nativeschema.MethodGetObject, - SenderKey: g.reqInfo.SenderKey, - ContainerOwner: g.reqInfo.ContainerOwner, - Role: g.reqInfo.Role, - BearerToken: g.metadata.BearerToken, - XHeaders: resp.GetMetaHeader().GetXHeaders(), + Namespace: g.reqInfo.Namespace, + Container: cnrID, + Object: objID, + Header: partInit.GetHeader(), + Method: nativeschema.MethodGetObject, + SenderKey: g.reqInfo.SenderKey, + ContainerOwner: g.reqInfo.ContainerOwner, + ContainerAttributes: g.reqInfo.ContainerAttributes, + Role: g.reqInfo.Role, + BearerToken: g.metadata.BearerToken, + XHeaders: resp.GetMetaHeader().GetXHeaders(), } if err := g.apeChecker.CheckAPE(g.Context(), prm); err != nil { @@ -142,16 +143,17 @@ func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutR } prm := Prm{ - Namespace: reqInfo.Namespace, - Container: md.Container, - Object: md.Object, - Header: partInit.GetHeader(), - Method: nativeschema.MethodPutObject, - SenderKey: reqInfo.SenderKey, - ContainerOwner: reqInfo.ContainerOwner, - Role: reqInfo.Role, - BearerToken: md.BearerToken, - XHeaders: md.MetaHeader.GetXHeaders(), + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, + Header: partInit.GetHeader(), + Method: nativeschema.MethodPutObject, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + ContainerAttributes: reqInfo.ContainerAttributes, + Role: reqInfo.Role, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -200,15 +202,16 @@ func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.Pa } prm := Prm{ - Namespace: reqInfo.Namespace, - Container: md.Container, - Object: md.Object, - Method: nativeschema.MethodPatchObject, - SenderKey: reqInfo.SenderKey, - ContainerOwner: reqInfo.ContainerOwner, - Role: reqInfo.Role, - BearerToken: md.BearerToken, - XHeaders: md.MetaHeader.GetXHeaders(), + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, + Method: nativeschema.MethodPatchObject, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + ContainerAttributes: reqInfo.ContainerAttributes, + Role: reqInfo.Role, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -268,16 +271,17 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqInfo.Namespace, - Container: md.Container, - Object: md.Object, - Header: header, - Method: nativeschema.MethodHeadObject, - Role: reqInfo.Role, - SenderKey: reqInfo.SenderKey, - ContainerOwner: reqInfo.ContainerOwner, - BearerToken: md.BearerToken, - XHeaders: md.MetaHeader.GetXHeaders(), + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, + Header: header, + Method: nativeschema.MethodHeadObject, + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + ContainerAttributes: reqInfo.ContainerAttributes, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), }) if err != nil { return nil, toStatusErr(err) @@ -296,14 +300,15 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqInfo.Namespace, - Container: md.Container, - Method: nativeschema.MethodSearchObject, - Role: reqInfo.Role, - SenderKey: reqInfo.SenderKey, - ContainerOwner: reqInfo.ContainerOwner, - BearerToken: md.BearerToken, - XHeaders: md.MetaHeader.GetXHeaders(), + Namespace: reqInfo.Namespace, + Container: md.Container, + Method: nativeschema.MethodSearchObject, + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + ContainerAttributes: reqInfo.ContainerAttributes, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), }) if err != nil { return toStatusErr(err) @@ -323,15 +328,16 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqInfo.Namespace, - Container: md.Container, - Object: md.Object, - Method: nativeschema.MethodDeleteObject, - Role: reqInfo.Role, - SenderKey: reqInfo.SenderKey, - ContainerOwner: reqInfo.ContainerOwner, - BearerToken: md.BearerToken, - XHeaders: md.MetaHeader.GetXHeaders(), + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, + Method: nativeschema.MethodDeleteObject, + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + ContainerAttributes: reqInfo.ContainerAttributes, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), }) if err != nil { return nil, toStatusErr(err) @@ -356,15 +362,16 @@ func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.G } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqInfo.Namespace, - Container: md.Container, - Object: md.Object, - Method: nativeschema.MethodRangeObject, - Role: reqInfo.Role, - SenderKey: reqInfo.SenderKey, - ContainerOwner: reqInfo.ContainerOwner, - BearerToken: md.BearerToken, - XHeaders: md.MetaHeader.GetXHeaders(), + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, + Method: nativeschema.MethodRangeObject, + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + ContainerAttributes: reqInfo.ContainerAttributes, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), }) if err != nil { return toStatusErr(err) @@ -384,15 +391,16 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa } prm := Prm{ - Namespace: reqInfo.Namespace, - Container: md.Container, - Object: md.Object, - Method: nativeschema.MethodHashObject, - Role: reqInfo.Role, - SenderKey: reqInfo.SenderKey, - ContainerOwner: reqInfo.ContainerOwner, - BearerToken: md.BearerToken, - XHeaders: md.MetaHeader.GetXHeaders(), + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, + Method: nativeschema.MethodHashObject, + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + ContainerAttributes: reqInfo.ContainerAttributes, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), } resp, err := c.next.GetRangeHash(ctx, request) @@ -417,16 +425,17 @@ func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequ } prm := Prm{ - Namespace: reqInfo.Namespace, - Container: md.Container, - Object: md.Object, - Header: request.GetBody().GetObject().GetHeader(), - Method: nativeschema.MethodPutObject, - Role: reqInfo.Role, - SenderKey: reqInfo.SenderKey, - ContainerOwner: reqInfo.ContainerOwner, - BearerToken: md.BearerToken, - XHeaders: md.MetaHeader.GetXHeaders(), + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, + Header: request.GetBody().GetObject().GetHeader(), + Method: nativeschema.MethodPutObject, + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + ContainerAttributes: reqInfo.ContainerAttributes, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { From 86aec1ad6dfd2404a385349d1937fa6af4488780 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 24 Apr 2025 19:08:30 +0300 Subject: [PATCH 92/94] [#1689] container: Make APE middleware form container system attributes * Make `Put` handler extract container attributes from request body and form APE-resource properties; * Make `validateContainerBoundedOperation` used by `Get` and `Delete` handlers extracts attributes from read container. Change-Id: I005345575c3d25b505bae4108f60cd320a7489ba Signed-off-by: Airat Arifullin --- pkg/services/container/ape.go | 29 ++++- pkg/services/container/ape_test.go | 181 +++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 4 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 01bd825d7..3b5dab9aa 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -280,11 +280,16 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont return nil, err } + cnrProps, err := getContainerPropsFromV2(req.GetBody().GetContainer()) + if err != nil { + return nil, fmt.Errorf("get container properties: %w", err) + } + request := aperequest.NewRequest( nativeschema.MethodPutContainer, aperequest.NewResource( resourceName(namespace, ""), - make(map[string]string), + cnrProps, ), reqProps, ) @@ -395,7 +400,7 @@ func (ac *apeChecker) validateContainerBoundedOperation(ctx context.Context, con op, aperequest.NewResource( resourceName(namespace, id.EncodeToString()), - ac.getContainerProps(cont), + getContainerProps(cont), ), reqProps, ) @@ -445,10 +450,26 @@ func resourceName(namespace string, container string) string { return fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainer, namespace, container) } -func (ac *apeChecker) getContainerProps(c *containercore.Container) map[string]string { - return map[string]string{ +func getContainerProps(c *containercore.Container) map[string]string { + props := map[string]string{ nativeschema.PropertyKeyContainerOwnerID: c.Value.Owner().EncodeToString(), } + for attrName, attrVal := range c.Value.Attributes() { + name := fmt.Sprintf(nativeschema.PropertyKeyFormatContainerAttribute, attrName) + props[name] = attrVal + } + return props +} + +func getContainerPropsFromV2(cnrV2 *container.Container) (map[string]string, error) { + if cnrV2 == nil { + return nil, errors.New("container is not set") + } + c := cnrSDK.Container{} + if err := c.ReadFromV2(*cnrV2); err != nil { + return nil, err + } + return getContainerProps(&containercore.Container{Value: c}), nil } func (ac *apeChecker) getRequestProps(ctx context.Context, mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index 77a981d1a..6438c34ca 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -54,6 +54,8 @@ func TestAPE(t *testing.T) { t.Run("deny put container with invlaid namespace", testDenyPutContainerInvalidNamespace) t.Run("deny list containers for owner with PK", testDenyListContainersForPK) t.Run("deny list containers by namespace invalidation", testDenyListContainersValidationNamespaceError) + t.Run("deny get by container attribute rules", testDenyGetContainerSysZoneAttr) + t.Run("deny put by container attribute rules", testDenyPutContainerSysZoneAttr) } const ( @@ -564,6 +566,185 @@ func testDenyGetContainerByIP(t *testing.T) { require.Contains(t, errAccessDenied.Reason(), chain.AccessDenied.String()) } +func testDenyGetContainerSysZoneAttr(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + pk.PublicKey().GetScriptHash(): { + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + }, + }, + subjectsExt: map[util.Uint160]*client.SubjectExtended{ + pk.PublicKey().GetScriptHash(): { + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 19888, + }, + }, + }, + }, + } + + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + testContainer.SetAttribute(container.SysAttributeZone, "eggplant") + contRdr.c[contID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Kind: chain.KindResource, + Key: fmt.Sprintf(nativeschema.PropertyKeyFormatContainerAttribute, container.SysAttributeZone), + Value: "eggplant", + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := &container.GetRequest{} + req.SetBody(&container.GetRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetContainerID(&refContID) + + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + resp, err := apeSrv.Get(ctxWithPeerInfo(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) + require.Contains(t, errAccessDenied.Reason(), chain.AccessDenied.String()) +} + +func testDenyPutContainerSysZoneAttr(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + testContainer.SetAttribute(container.SysAttributeZone, "eggplant") + contRdr.c[contID] = &containercore.Container{Value: testContainer} + owner := testContainer.Owner() + ownerAddr := owner.ScriptHash() + + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + ownerAddr: {}, + }, + subjectsExt: map[util.Uint160]*client.SubjectExtended{ + ownerAddr: {}, + }, + } + + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(""), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodPutContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + nativeschema.ResourceFormatRootContainers, + }, + }, + Condition: []chain.Condition{ + { + Kind: chain.KindResource, + Key: fmt.Sprintf(nativeschema.PropertyKeyFormatContainerAttribute, container.SysAttributeZone), + Value: "eggplant", + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := initPutRequest(t, testContainer) + + resp, err := apeSrv.Put(ctxWithPeerInfo(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) + require.Contains(t, errAccessDenied.Reason(), chain.AccessDenied.String()) +} + func testDenyGetContainerByGroupID(t *testing.T) { t.Parallel() srv := &srvStub{ From 0b5ac445b676c123156648d2274d7565d7908dd0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 May 2025 10:54:38 +0300 Subject: [PATCH 93/94] [#1689] engine: Use IO tag critical for engine reload Storage engine reload may cause IO operations (writecache flush), that should not be scheduled in some way. Change-Id: I7486a4646b4dd9bcedd2aa432b749e7f206b8b77 Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/control.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index bf1649f6e..39e532b6b 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -9,8 +9,10 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) @@ -211,6 +213,7 @@ func (rCfg *ReConfiguration) AddShard(id string, opts []shard.Option) { // Reload reloads StorageEngine's configuration in runtime. func (e *StorageEngine) Reload(ctx context.Context, rcfg ReConfiguration) error { + ctx = tagging.ContextWithIOTag(ctx, qos.IOTagCritical.String()) type reloadInfo struct { sh *shard.Shard opts []shard.Option From b01f05c951eeb97ef0e63a63de324e644fad8958 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 May 2025 12:08:38 +0300 Subject: [PATCH 94/94] [#1689] shard: Use single qos.Limiter instance for shard and writecache Before fix shard and writecache used the same instance of qos.Limiter. In case of SIGHUP signal shard was closing qos.Limiter, but didn't update writecache's pointer. Now shard uses atomic pointer to qos.Limiter and shares it with writecache. On SIGHUP shard updates atomic pointer value and closes old qos.Limiter. Change-Id: Ic2ab62441d3872e71c5771f54d070e0ca48fe375 Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 - pkg/local_object_storage/shard/control.go | 17 +++++-- pkg/local_object_storage/shard/shard.go | 56 +++++++++++++++++++---- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 96274e625..83a9b4d4c 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -920,7 +920,6 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { writecache.WithMaxCacheCount(wcRead.countLimit), writecache.WithNoSync(wcRead.noSync), writecache.WithLogger(c.log.WithTag(logger.TagWriteCache)), - writecache.WithQoSLimiter(shCfg.limiter), ) } return writeCacheOpts diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index d489b8b0d..a607f70f7 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -452,11 +452,20 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { return err } } - if c.opsLimiter != nil { - s.opsLimiter.Close() - s.opsLimiter = c.opsLimiter + if err := s.setMode(ctx, c.info.Mode); err != nil { + return err + } + s.reloadOpsLimiter(&c) + + return nil +} + +func (s *Shard) reloadOpsLimiter(c *cfg) { + if c.configOpsLimiter != nil { + old := s.opsLimiter.ptr.Swap(&qosLimiterHolder{Limiter: c.configOpsLimiter}) + old.Close() + s.opsLimiter.SetParentID(s.info.ID.String()) } - return s.setMode(ctx, c.info.Mode) } func (s *Shard) lockExclusive() func() { diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index d89b56266..f21541d9d 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -39,6 +39,8 @@ type Shard struct { rb *rebuilder + opsLimiter *atomicOpsLimiter + gcCancel atomic.Value setModeRequested atomic.Bool writecacheSealCancel atomic.Pointer[writecacheSealCanceler] @@ -100,7 +102,7 @@ type cfg struct { containerInfo container.InfoProvider - opsLimiter qos.Limiter + configOpsLimiter qos.Limiter } func defaultCfg() *cfg { @@ -112,7 +114,7 @@ func defaultCfg() *cfg { zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, zeroCountContainersCallback: func(context.Context, []cid.ID) {}, metricsWriter: noopMetrics{}, - opsLimiter: qos.NewNoopLimiter(), + configOpsLimiter: qos.NewNoopLimiter(), } } @@ -128,10 +130,11 @@ func New(opts ...Option) *Shard { mb := meta.New(c.metaOpts...) s := &Shard{ - cfg: c, - blobStor: bs, - metaBase: mb, - tsSource: c.tsSource, + cfg: c, + blobStor: bs, + metaBase: mb, + tsSource: c.tsSource, + opsLimiter: newAtomicOpsLimiter(c.configOpsLimiter), } reportFunc := func(ctx context.Context, msg string, err error) { @@ -145,7 +148,8 @@ func New(opts ...Option) *Shard { append(c.writeCacheOpts, writecache.WithReportErrorFunc(reportFunc), writecache.WithBlobstor(bs), - writecache.WithMetabase(mb))...) + writecache.WithMetabase(mb), + writecache.WithQoSLimiter(s.opsLimiter))...) s.writeCache.GetMetrics().SetPath(s.writeCache.DumpInfo().Path) } @@ -374,7 +378,7 @@ func WithContainerInfoProvider(containerInfo container.InfoProvider) Option { func WithLimiter(l qos.Limiter) Option { return func(c *cfg) { - c.opsLimiter = l + c.configOpsLimiter = l } } @@ -520,3 +524,39 @@ func (s *Shard) SetEvacuationInProgress(val bool) { s.info.EvacuationInProgress = val s.metricsWriter.SetEvacuationInProgress(val) } + +var _ qos.Limiter = &atomicOpsLimiter{} + +func newAtomicOpsLimiter(l qos.Limiter) *atomicOpsLimiter { + result := &atomicOpsLimiter{} + result.ptr.Store(&qosLimiterHolder{Limiter: l}) + return result +} + +type atomicOpsLimiter struct { + ptr atomic.Pointer[qosLimiterHolder] +} + +func (a *atomicOpsLimiter) Close() { + a.ptr.Load().Close() +} + +func (a *atomicOpsLimiter) ReadRequest(ctx context.Context) (qos.ReleaseFunc, error) { + return a.ptr.Load().ReadRequest(ctx) +} + +func (a *atomicOpsLimiter) SetMetrics(m qos.Metrics) { + a.ptr.Load().SetMetrics(m) +} + +func (a *atomicOpsLimiter) SetParentID(id string) { + a.ptr.Load().SetParentID(id) +} + +func (a *atomicOpsLimiter) WriteRequest(ctx context.Context) (qos.ReleaseFunc, error) { + return a.ptr.Load().WriteRequest(ctx) +} + +type qosLimiterHolder struct { + qos.Limiter +}