[#1439] object: Sort nodes by priority metrics to compute GET/SEARCH requests

Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>

.docker/Dockerfile.adm

@@ -1,4 +1,4 @@
-FROM golang:1.22 AS builder
+FROM golang:1.23 AS builder
 ARG BUILD=now
 ARG VERSION=dev
 ARG REPO=repository

.docker/Dockerfile.ci

@@ -1,4 +1,4 @@
-FROM golang:1.22
+FROM golang:1.23
 
 WORKDIR /tmp
 

.docker/Dockerfile.cli

@@ -1,4 +1,4 @@
-FROM golang:1.22 AS builder
+FROM golang:1.23 AS builder
 ARG BUILD=now
 ARG VERSION=dev
 ARG REPO=repository

.docker/Dockerfile.ir

@@ -1,4 +1,4 @@
-FROM golang:1.22 AS builder
+FROM golang:1.23 AS builder
 ARG BUILD=now
 ARG VERSION=dev
 ARG REPO=repository

.docker/Dockerfile.storage

@@ -1,4 +1,4 @@
-FROM golang:1.22 AS builder
+FROM golang:1.23 AS builder
 ARG BUILD=now
 ARG VERSION=dev
 ARG REPO=repository

Makefile

@@ -27,12 +27,6 @@ DIRS = $(BIN) $(RELEASE)
 CMDS = $(notdir $(basename $(wildcard cmd/frostfs-*)))
 BINS = $(addprefix $(BIN)/, $(CMDS))
 
-# .deb package versioning
-OS_RELEASE = $(shell lsb_release -cs)
-PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
-			sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
-			sed "s/-/~/")-${OS_RELEASE}
-
 OUTPUT_LINT_DIR ?= $(abspath $(BIN))/linters
 LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
 TMP_DIR := .cache
@@ -58,7 +52,7 @@ LOCODE_DB_PATH=$(abspath ./.cache/locode_db)
 LOCODE_DB_VERSION=v0.4.0
 
 .PHONY: help all images dep clean fmts fumpt imports test lint docker/lint
-		prepare-release debpackage pre-commit unpre-commit
+		prepare-release pre-commit unpre-commit
 
 # To build a specific binary, use it's name prefix with bin/ as a target
 # For example `make bin/frostfs-node` will build only storage node binary
@@ -263,19 +257,6 @@ clean:
 	rm -rf $(BIN)
 	rm -rf $(RELEASE)
 
-# Package for Debian
-debpackage:
-	dch -b --package frostfs-node \
-			--controlmaint \
-			--newversion $(PKG_VERSION) \
-			--distribution $(OS_RELEASE) \
-			"Please see CHANGELOG.md for code changes for $(VERSION)"
-	dpkg-buildpackage --no-sign -b
-
-# Cleanup deb package build directories
-debclean:
-	dh clean
-
 # Download locode database
 locode-download:
 	mkdir -p $(TMP_DIR)

cmd/frostfs-node/config.go

@@ -58,6 +58,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
 	objectService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object"
 	getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/tombstone"
 	tsourse "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/tombstone/source"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator"
@@ -109,6 +110,7 @@ type applicationConfiguration struct {
 
 	ObjectCfg struct {
 		tombstoneLifetime uint64
+		priorityMetrics   []placement.Metric
 	}
 
 	EngineCfg struct {
@@ -232,6 +234,11 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error {
 	// Object
 
 	a.ObjectCfg.tombstoneLifetime = objectconfig.TombstoneLifetime(c)
+	var pm []placement.Metric
+	for _, raw := range objectconfig.Get(c).Priority() {
+		pm = append(pm, placement.ParseMetric(raw))
+	}
+	a.ObjectCfg.priorityMetrics = pm
 
 	// Storage Engine
 

cmd/frostfs-node/config/object/config.go

@@ -10,10 +10,17 @@ type PutConfig struct {
 	cfg *config.Config
 }
 
+// GetConfig is a wrapper over "get" config section which provides access
+// to object get pipeline configuration of object service.
+type GetConfig struct {
+	cfg *config.Config
+}
+
 const (
 	subsection = "object"
 
 	putSubsection = "put"
+	getSubsection = "get"
 
 	// PutPoolSizeDefault is a default value of routine pool size to
 	// process object.Put requests in object service.
@@ -56,3 +63,16 @@ func (g PutConfig) PoolSizeLocal() int {
 func (g PutConfig) SkipSessionTokenIssuerVerification() bool {
 	return config.BoolSafe(g.cfg, "skip_session_token_issuer_verification")
 }
+
+// Get returns structure that provides access to "get" subsection of
+// "object" section.
+func Get(c *config.Config) GetConfig {
+	return GetConfig{
+		c.Sub(subsection).Sub(getSubsection),
+	}
+}
+
+// Priority returns the value of "priority" config parameter.
+func (g GetConfig) Priority() []string {
+	return config.StringSliceSafe(g.cfg, "priority")
+}

cmd/frostfs-node/object.go

@@ -174,11 +174,13 @@ func initObjectService(c *cfg) {
 
 	sPutV2 := createPutSvcV2(sPut, keyStorage)
 
-	sSearch := createSearchSvc(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource)
+	sSearch := createSearchSvc(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource,
+		c.ObjectCfg.priorityMetrics)
 
 	sSearchV2 := createSearchSvcV2(sSearch, keyStorage)
 
-	sGet := createGetService(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource)
+	sGet := createGetService(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource,
+		c.ObjectCfg.priorityMetrics)
 
 	*c.cfgObject.getSvc = *sGet // need smth better
 
@@ -366,7 +368,10 @@ func createPatchSvc(sGet *getsvc.Service, sPut *putsvc.Service) *patchsvc.Servic
 	return patchsvc.NewService(sPut.Config, sGet)
 }
 
-func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache, containerSource containercore.Source) *searchsvc.Service {
+func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache,
+	containerSource containercore.Source,
+	priorityMetrics []placement.Metric,
+) *searchsvc.Service {
 	ls := c.cfgObject.cfgLocalStorage.localStorage
 
 	return searchsvc.New(
@@ -374,6 +379,8 @@ func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Trav
 		coreConstructor,
 		traverseGen.WithTraverseOptions(
 			placement.WithoutSuccessTracking(),
+			placement.WithPriorityMetrics(priorityMetrics),
+			placement.WithNodeState(c),
 		),
 		c.netMapSource,
 		keyStorage,
@@ -389,6 +396,7 @@ func createSearchSvcV2(sSearch *searchsvc.Service, keyStorage *util.KeyStorage)
 func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator,
 	coreConstructor *cache.ClientCache,
 	containerSource containercore.Source,
+	priorityMetrics []placement.Metric,
 ) *getsvc.Service {
 	ls := c.cfgObject.cfgLocalStorage.localStorage
 
@@ -398,6 +406,8 @@ func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Tra
 		ls,
 		traverseGen.WithTraverseOptions(
 			placement.SuccessAfter(1),
+			placement.WithPriorityMetrics(priorityMetrics),
+			placement.WithNodeState(c),
 		),
 		coreConstructor,
 		containerSource,

cmd/frostfs-node/validate.go

@@ -8,9 +8,11 @@ import (
 	engineconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine"
 	shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard"
 	loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger"
+	objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object"
 	treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
 )
 
@@ -30,6 +32,13 @@ func validateConfig(c *config.Config) error {
 		return fmt.Errorf("invalid logger destination: %w", err)
 	}
 
+	// validate priority metrics for GET and SEARCH requests
+	for _, raw := range objectconfig.Get(c).Priority() {
+		if err := placement.ValidateMetric(raw); err != nil {
+			return err
+		}
+	}
+
 	// shard configuration validation
 
 	shardNum := 0

config/example/node.env

@@ -87,6 +87,7 @@ FROSTFS_OBJECT_PUT_REMOTE_POOL_SIZE=100
 FROSTFS_OBJECT_PUT_LOCAL_POOL_SIZE=200
 FROSTFS_OBJECT_PUT_SKIP_SESSION_TOKEN_ISSUER_VERIFICATION=true
 FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10
+FROSTFS_OBJECT_GET_PRIORITY="$attribute:ClusterName $attribute:UN-LOCODE"
 
 # Storage engine section
 FROSTFS_STORAGE_SHARD_POOL_SIZE=15

config/example/node.json

@@ -131,6 +131,9 @@
       "remote_pool_size": 100,
       "local_pool_size": 200,
       "skip_session_token_issuer_verification": true
+    },
+    "get": {
+      "priority": ["$attribute:ClusterName", "$attribute:UN-LOCODE"]
     }
   },
   "storage": {

config/example/node.yaml

@@ -114,6 +114,10 @@ object:
     remote_pool_size: 100  # number of async workers for remote PUT operations
     local_pool_size: 200  # number of async workers for local PUT operations
     skip_session_token_issuer_verification: true # session token issuer verification will be skipped if true
+  get:
+    priority:  # list of metrics of nodes for prioritization
+      - $attribute:ClusterName
+      - $attribute:UN-LOCODE
 
 storage:
   # note: shard configuration can be omitted for relay node (see `node.relay`)

debian/changelog

@@ -1,5 +0,0 @@
-frostfs-node (0.0.1) stable; urgency=medium
-
-  * Initial package build
-
- -- TrueCloudLab <tech@frostfs.info>  Tue, 25 Oct 2022 21:10:49 +0300

debian/clean

@@ -1,2 +0,0 @@
-man/
-debian/*.bash-completion

debian/control

@@ -1,39 +0,0 @@
-Source: frostfs-node
-Section: misc
-Priority: optional
-Maintainer: TrueCloudLab <tech@frostfs.info>
-Build-Depends: debhelper-compat (= 13), dh-sequence-bash-completion, devscripts
-Standards-Version: 4.5.1
-Homepage: https://fs.neo.org/
-Vcs-Git: https://git.frostfs.info/TrueCloudLab/frostfs-node.git
-Vcs-Browser: https://git.frostfs.info/TrueCloudLab/frostfs-node
-
-Package: frostfs-storage
-Architecture: any
-Depends: ${misc:Depends}
-Description: FrostFS Storage node
- FrostFS is a decentralized distributed object storage integrated with the NEO
- Blockchain. FrostFS Nodes are organized in a peer-to-peer network that takes care
- of storing and distributing user's data. Any Neo user may participate in the
- network and get paid for providing storage resources to other users or store
- their data in FrostFS and pay a competitive price for it.
-
-Package: frostfs-ir
-Architecture: any
-Depends: ${misc:Depends}, frostfs-locode-db
-Description: FrostFS InnerRing node
- FrostFS is a decentralized distributed object storage integrated with the NEO
- Blockchain. FrostFS Nodes are organized in a peer-to-peer network that takes care
- of storing and distributing user's data. Any Neo user may participate in the
- network and get paid for providing storage resources to other users or store
- their data in FrostFS and pay a competitive price for it.
-
-Package: frostfs-cli
-Architecture: any
-Depends: ${misc:Depends}
-Description: CLI tools for FrostFS
- FrostFS is a decentralized distributed object storage integrated with the NEO
- Blockchain. FrostFS Nodes are organized in a peer-to-peer network that takes care
- of storing and distributing user's data. Any Neo user may participate in the
- network and get paid for providing storage resources to other users or store
- their data in FrostFS and pay a competitive price for it.

debian/copyright

@@ -1,23 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: frostfs-node
-Upstream-Contact: tech@frostfs.info
-Source: https://git.frostfs.info/TrueCloudLab/frostfs-node
-
-Files: *
-Copyright: 2022-2023 TrueCloudLab (@TrueCloudLab), contributors of FrostFS project
-           2018-2022 NeoSPCC (@nspcc-dev), contributors of NeoFS project
-           (https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/CREDITS.md)
-
-License: GPL-3
- This program is free software: you can redistribute it and/or modify it
- under the terms of the GNU General Public License as published
- by the Free Software Foundation; version 3.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program or at /usr/share/common-licenses/GPL-3
- If not, see <http://www.gnu.org/licenses/>.

debian/frostfs-cli.docs

@@ -1,4 +0,0 @@
-CONTRIBUTING.md
-CREDITS.md
-README.md
-cmd/frostfs-adm/docs

debian/frostfs-cli.install

@@ -1,3 +0,0 @@
-bin/frostfs-adm   usr/bin
-bin/frostfs-cli   usr/bin
-bin/frostfs-lens  usr/bin

debian/frostfs-cli.manpages

@@ -1 +0,0 @@
-man/*

debian/frostfs-ir.dirs

@@ -1,2 +0,0 @@
-/etc/frostfs/ir
-/var/lib/frostfs/ir

debian/frostfs-ir.docs

@@ -1,3 +0,0 @@
-CONTRIBUTING.md
-CREDITS.md
-README.md

debian/frostfs-ir.install

@@ -1 +0,0 @@
-bin/frostfs-ir    usr/bin

debian/frostfs-ir.postinst

@@ -1,51 +0,0 @@
-#!/bin/sh
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-#        * <postinst> `configure' <most-recently-configured-version>
-#        * <old-postinst> `abort-upgrade' <new version>
-#        * <conflictor's-postinst> `abort-remove' `in-favour' <package>
-#          <new-version>
-#        * <postinst> `abort-remove'
-#        * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
-#          <failed-install-package> <version> `removing'
-#          <conflicting-package> <version>
-# for details, see https://www.debian.org/doc/debian-policy/ or
-# the debian-policy package
-
-case "$1" in
-    configure)
-        USERNAME=ir
-        id -u frostfs-ir >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /var/lib/frostfs/ir --system -M -U -c "FrostFS InnerRing node" frostfs-ir
-        if ! dpkg-statoverride --list /etc/frostfs/$USERNAME >/dev/null; then
-            chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME
-            chmod -f 0750 /etc/frostfs/$USERNAME
-            chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/config.yml
-            chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/control.yml
-            chmod -f 0640 /etc/frostfs/$USERNAME/config.yml || true
-            chmod -f 0640 /etc/frostfs/$USERNAME/control.yml || true
-        fi
-        USERDIR="$(getent passwd frostfs-$USERNAME | cut -d: -f6)"
-        if ! dpkg-statoverride --list frostfs-"$USERDIR" >/dev/null; then
-            chown -f frostfs-$USERNAME: "$USERDIR"
-        fi
-    ;;
-
-    abort-upgrade|abort-remove|abort-deconfigure)
-    ;;
-
-    *)
-        echo "postinst called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0

debian/frostfs-ir.postrm

@@ -1,40 +0,0 @@
-#!/bin/sh
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-#        * <postrm> `remove'
-#        * <postrm> `purge'
-#        * <old-postrm> `upgrade' <new-version>
-#        * <new-postrm> `failed-upgrade' <old-version>
-#        * <new-postrm> `abort-install'
-#        * <new-postrm> `abort-install' <old-version>
-#        * <new-postrm> `abort-upgrade' <old-version>
-#        * <disappearer's-postrm> `disappear' <overwriter>
-#          <overwriter-version>
-# for details, see https://www.debian.org/doc/debian-policy/ or
-# the debian-policy package
-
-
-case "$1" in
-    purge)
-        rm -rf /var/lib/frostfs/ir/*
-    ;;
-
-    remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
-    ;;
-
-    *)
-        echo "postrm called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0

debian/frostfs-ir.preinst

@@ -1,34 +0,0 @@
-#!/bin/sh
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-#        * <new-preinst> `install'
-#        * <new-preinst> `install' <old-version>
-#        * <new-preinst> `upgrade' <old-version>
-#        * <old-preinst> `abort-upgrade' <new-version>
-# for details, see https://www.debian.org/doc/debian-policy/ or
-# the debian-policy package
-
-
-case "$1" in
-    install|upgrade)
-    ;;
-
-    abort-upgrade)
-    ;;
-
-    *)
-        echo "preinst called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0

debian/frostfs-ir.prerm

@@ -1,37 +0,0 @@
-#!/bin/sh
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-#        * <prerm> `remove'
-#        * <old-prerm> `upgrade' <new-version>
-#        * <new-prerm> `failed-upgrade' <old-version>
-#        * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
-#        * <deconfigured's-prerm> `deconfigure' `in-favour'
-#          <package-being-installed> <version> `removing'
-#          <conflicting-package> <version>
-# for details, see https://www.debian.org/doc/debian-policy/ or
-# the debian-policy package
-
-
-case "$1" in
-    remove|upgrade|deconfigure)
-    ;;
-
-    failed-upgrade)
-    ;;
-
-    *)
-        echo "prerm called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0

debian/frostfs-ir.service

@@ -1,17 +0,0 @@
-[Unit]
-Description=FrostFS InnerRing node
-Requires=network.target
-
-[Service]
-Type=notify
-NotifyAccess=all
-ExecStart=/usr/bin/frostfs-ir --config /etc/frostfs/ir/config.yml
-User=frostfs-ir
-Group=frostfs-ir
-WorkingDirectory=/var/lib/frostfs/ir
-Restart=always
-RestartSec=5
-PrivateTmp=true
-
-[Install]
-WantedBy=multi-user.target

debian/frostfs-storage.dirs

@@ -1,3 +0,0 @@
-/etc/frostfs/storage
-/srv/frostfs
-/var/lib/frostfs/storage

debian/frostfs-storage.docs

@@ -1,4 +0,0 @@
-docs/storage-node-configuration.md
-CONTRIBUTING.md
-CREDITS.md
-README.md

debian/frostfs-storage.install

@@ -1 +0,0 @@
-bin/frostfs-node  usr/bin

debian/frostfs-storage.postinst

@@ -1,55 +0,0 @@
-#!/bin/sh
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-#        * <postinst> `configure' <most-recently-configured-version>
-#        * <old-postinst> `abort-upgrade' <new version>
-#        * <conflictor's-postinst> `abort-remove' `in-favour' <package>
-#          <new-version>
-#        * <postinst> `abort-remove'
-#        * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
-#          <failed-install-package> <version> `removing'
-#          <conflicting-package> <version>
-# for details, see https://www.debian.org/doc/debian-policy/ or
-# the debian-policy package
-
-case "$1" in
-    configure)
-        USERNAME=storage
-        id -u frostfs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /var/lib/frostfs/$USERNAME --system -M -U -c "FrostFS Storage node" frostfs-$USERNAME
-        if ! dpkg-statoverride --list /etc/frostfs/$USERNAME >/dev/null; then
-            chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME
-            chmod -f 0750 /etc/frostfs/$USERNAME
-            chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/config.yml
-            chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/control.yml
-            chmod -f 0640 /etc/frostfs/$USERNAME/config.yml || true
-            chmod -f 0640 /etc/frostfs/$USERNAME/control.yml || true
-        fi
-        USERDIR=$(getent passwd frostfs-$USERNAME | cut -d: -f6)
-        if ! dpkg-statoverride --list frostfs-"$USERDIR" >/dev/null; then
-            chown -f frostfs-$USERNAME: "$USERDIR"
-        fi
-        USERDIR=/srv/frostfs
-        if ! dpkg-statoverride --list frostfs-$USERDIR >/dev/null; then
-            chown -f frostfs-$USERNAME: $USERDIR
-        fi
-    ;;
-
-    abort-upgrade|abort-remove|abort-deconfigure)
-    ;;
-
-    *)
-        echo "postinst called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0

debian/frostfs-storage.postrm

@@ -1,40 +0,0 @@
-#!/bin/sh
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-#        * <postrm> `remove'
-#        * <postrm> `purge'
-#        * <old-postrm> `upgrade' <new-version>
-#        * <new-postrm> `failed-upgrade' <old-version>
-#        * <new-postrm> `abort-install'
-#        * <new-postrm> `abort-install' <old-version>
-#        * <new-postrm> `abort-upgrade' <old-version>
-#        * <disappearer's-postrm> `disappear' <overwriter>
-#          <overwriter-version>
-# for details, see https://www.debian.org/doc/debian-policy/ or
-# the debian-policy package
-
-
-case "$1" in
-    purge)
-        rm -rf /var/lib/frostfs/storage/*
-    ;;
-
-    remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
-    ;;
-
-    *)
-        echo "postrm called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0

debian/frostfs-storage.preinst

@@ -1,34 +0,0 @@
-#!/bin/sh
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-#        * <new-preinst> `install'
-#        * <new-preinst> `install' <old-version>
-#        * <new-preinst> `upgrade' <old-version>
-#        * <old-preinst> `abort-upgrade' <new-version>
-# for details, see https://www.debian.org/doc/debian-policy/ or
-# the debian-policy package
-
-
-case "$1" in
-    install|upgrade)
-    ;;
-
-    abort-upgrade)
-    ;;
-
-    *)
-        echo "preinst called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0

debian/frostfs-storage.prerm

@@ -1,37 +0,0 @@
-#!/bin/sh
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-#        * <prerm> `remove'
-#        * <old-prerm> `upgrade' <new-version>
-#        * <new-prerm> `failed-upgrade' <old-version>
-#        * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
-#        * <deconfigured's-prerm> `deconfigure' `in-favour'
-#          <package-being-installed> <version> `removing'
-#          <conflicting-package> <version>
-# for details, see https://www.debian.org/doc/debian-policy/ or
-# the debian-policy package
-
-
-case "$1" in
-    remove|upgrade|deconfigure)
-    ;;
-
-    failed-upgrade)
-    ;;
-
-    *)
-        echo "prerm called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0

debian/frostfs-storage.service

@@ -1,17 +0,0 @@
-[Unit]
-Description=FrostFS Storage node
-Requires=network.target
-
-[Service]
-Type=notify
-NotifyAccess=all
-ExecStart=/usr/bin/frostfs-node --config /etc/frostfs/storage/config.yml
-User=frostfs-storage
-Group=frostfs-storage
-WorkingDirectory=/srv/frostfs
-Restart=always
-RestartSec=5
-PrivateTmp=true
-
-[Install]
-WantedBy=multi-user.target

debian/rules

@@ -1,40 +0,0 @@
-#!/usr/bin/make -f
-
-# Do not try to strip Go binaries
-export DEB_BUILD_OPTIONS := nostrip
-
-%:
-	dh $@ --with bash-completion
-
-override_dh_auto_test:
-
-override_dh_auto_install:
-	echo $(DEB_BUILD_OPTIONS)
-	dh_auto_install
-
-	bin/frostfs-adm gendoc --type man man/
-	bin/frostfs-cli gendoc --type man man/
-
-	bin/frostfs-adm completion bash > debian/frostfs-adm.bash-completion
-	bin/frostfs-cli completion bash > debian/frostfs-cli.bash-completion
-	install -m 0755 -d debian/frostfs-cli/usr/share/fish/completions/
-	install -m 0755 -d debian/frostfs-cli/usr/share/zsh/vendor-completions/
-	bin/frostfs-adm completion fish > debian/frostfs-cli/usr/share/fish/completions/frostfs-adm.fish
-	bin/frostfs-adm completion zsh > debian/frostfs-cli/usr/share/zsh/vendor-completions/_frostfs-adm
-	bin/frostfs-cli completion fish > debian/frostfs-cli/usr/share/fish/completions/frostfs-cli.fish
-	bin/frostfs-cli completion zsh > debian/frostfs-cli/usr/share/zsh/vendor-completions/_frostfs-cli
-
-	install -T -m 0640 config/example/ir.yaml debian/frostfs-ir/etc/frostfs/ir/config.yml
-	install -T -m 0640 config/example/ir-control.yaml debian/frostfs-ir/etc/frostfs/ir/control.yml
-	install -T -m 0640 config/example/node.yaml debian/frostfs-storage/etc/frostfs/storage/config.yml
-	install -T -m 0640 config/example/node-control.yaml debian/frostfs-storage/etc/frostfs/storage/control.yml
-
-override_dh_installsystemd:
-	dh_installsystemd --no-enable --no-start --name=frostfs-ir
-	dh_installsystemd --no-enable --no-start --name=frostfs-storage
-
-override_dh_installchangelogs:
-	dh_installchangelogs -k CHANGELOG.md
-
-override_dh_installdocs:
-	dh_installdocs

debian/source/format

@@ -1 +0,0 @@
-3.0 (quilt)

docs/building-deb-package.md

@@ -1,46 +0,0 @@
-# Building Debian package on host
-
-## Prerequisites
-
-For now, we're assuming building for Debian 11 (stable) x86_64.
-
-Go version 18.4 or later should already be installed, i.e. this runs
-successfully:
-
-* `make all`
-
-## Installing packaging dependencies
-
-```shell
-$ sudo apt install debhelper-compat dh-sequence-bash-completion devscripts
-```
-
-Warining: number of package installed is pretty large considering dependecies.
-
-## Package building
-
-```shell
-$ make debpackage
-```
-
-## Leftovers cleaning
-
-```shell
-$ make debclean
-```
-or
-```shell
-$ dh clean
-```
-
-# Package versioning
-
-By default, package version is based on product version and may also contain git
-tags and hashes.
-
-Package version could be overwritten by setting `PKG_VERSION` variable before
-build, Debian package versioning rules should be respected.
-
-```shell
-$ PKG_VERSION=0.32.0 make debpackge
-```

docs/release-instruction.md

@@ -43,11 +43,6 @@ Write new revision number into the root `VERSION` file:
 $ echo ${FROSTFS_TAG_PREFIX}${FROSTFS_REVISION} > VERSION
 ```
 
-Update version in Debian package changelog file
-```
-$ cat debian/changelog
-```
-
 Update the supported version of `TrueCloudLab/frostfs-contract` module in root
 `README.md` if needed.
 

docs/storage-node-configuration.md

@@ -407,13 +407,17 @@ Contains object-service related parameters.
 object:
   put:
     remote_pool_size: 100
+  get:
+    priority:
+      - $attribute:ClusterName
 ```
 
-| Parameter                   | Type  | Default value | Description                                                                                    |
+| Parameter                   | Type       | Default value | Description                                                                                          |
-|-----------------------------|-------|---------------|------------------------------------------------------------------------------------------------|
+|-----------------------------|------------|---------------|------------------------------------------------------------------------------------------------------|
-| `delete.tombstone_lifetime` | `int` | `5`           | Tombstone lifetime for removed objects in epochs.                                              |
+| `delete.tombstone_lifetime` | `int`      | `5`           | Tombstone lifetime for removed objects in epochs.                                                    |
-| `put.remote_pool_size`      | `int` | `10`          | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. |
+| `put.remote_pool_size`      | `int`      | `10`          | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services.       |
-| `put.local_pool_size`       | `int` | `10`          | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services.  |
+| `put.local_pool_size`       | `int`      | `10`          | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services.        |
+| `get.priority`              | `[]string` |               | List of metrics of nodes for prioritization. Used for computing response on GET and SEARCH requests. |
 
 # `runtime` section
 Contains runtime parameters.

pkg/services/object_manager/placement/metrics.go

@@ -0,0 +1,51 @@
+package placement
+
+import (
+	"errors"
+	"strings"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
+)
+
+const (
+	attrPrefix = "$attribute:"
+)
+
+type Metric interface {
+	CalculateValue(*netmap.NodeInfo, *netmap.NodeInfo) []byte
+}
+
+func ValidateMetric(raw string) error {
+	if strings.HasPrefix(raw, attrPrefix) {
+		return nil
+	}
+	return errors.New("unsupported priority metric")
+}
+
+func ParseMetric(raw string) Metric {
+	if attr, found := strings.CutPrefix(raw, attrPrefix); found {
+		return NewAttributeMetric(attr)
+	}
+	return nil
+}
+
+// attributeMetric describes priority metric based on attribute.
+type attributeMetric struct {
+	attribute string
+}
+
+// CalculateValue return [0] if from and to contains attribute attributeMetric.attribute and
+// the value of attribute is the same. In other case return [1].
+func (am *attributeMetric) CalculateValue(from *netmap.NodeInfo, to *netmap.NodeInfo) []byte {
+	fromAttr := from.Attribute(am.attribute)
+	toAttr := to.Attribute(am.attribute)
+	if len(fromAttr) > 0 && len(toAttr) > 0 && fromAttr == toAttr {
+		return []byte{0}
+	}
+	return []byte{1}
+}
+
+func NewAttributeMetric(raw string) Metric {
+	attr, _ := strings.CutPrefix(raw, attrPrefix)
+	return &attributeMetric{attribute: attr}
+}

pkg/services/object_manager/placement/traverser.go

@@ -1,10 +1,14 @@
 package placement
 
 import (
+	"bytes"
+	"encoding/binary"
 	"errors"
 	"fmt"
+	"slices"
 	"sync"
 
+	netmapAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@@ -23,6 +27,11 @@ type Builder interface {
 	BuildPlacement(cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error)
 }
 
+type NodeState interface {
+	// LocalNodeInfo return current node state in FrostFS API v2 NodeInfo structure.
+	LocalNodeInfo() (*netmapAPI.NodeInfo, error)
+}
+
 // Option represents placement traverser option.
 type Option func(*cfg)
 
@@ -50,9 +59,16 @@ type cfg struct {
 	policy    netmap.PlacementPolicy
 
 	builder Builder
+
+	metrics []Metric
+
+	nodeState NodeState
 }
 
-const invalidOptsMsg = "invalid traverser options"
+const (
+	invalidOptsMsg = "invalid traverser options"
+	uint16Bytes    = 2
+)
 
 var errNilBuilder = errors.New("placement builder is nil")
 
@@ -99,7 +115,26 @@ func NewTraverser(opts ...Option) (*Traverser, error) {
 	}
 
 	var rem []int
-	if cfg.flatSuccess != nil {
+	if len(cfg.metrics) > 0 {
+		rem = defaultCopiesVector(cfg.policy)
+		var unsortedVector []netmap.NodeInfo
+		var regularVector []netmap.NodeInfo
+		for i := range rem {
+			unsortedVector = append(unsortedVector, ns[i][:rem[i]]...)
+			regularVector = append(regularVector, ns[i][rem[i]:]...)
+		}
+		rem = make([]int, 2)
+		rem[0] = -1
+		rem[1] = -1
+
+		sortedVector, err := sortVector(cfg, unsortedVector)
+		if err != nil {
+			return nil, err
+		}
+		ns = make([][]netmap.NodeInfo, 2)
+		ns[0] = sortedVector
+		ns[1] = regularVector
+	} else if cfg.flatSuccess != nil {
 		ns = flatNodes(ns)
 		rem = []int{int(*cfg.flatSuccess)}
 	} else {
@@ -157,6 +192,37 @@ func flatNodes(ns [][]netmap.NodeInfo) [][]netmap.NodeInfo {
 	return [][]netmap.NodeInfo{flat}
 }
 
+func sortVector(cfg *cfg, unsortedVector []netmap.NodeInfo) ([]netmap.NodeInfo, error) {
+	metrics := make([][]byte, len(unsortedVector))
+	var node netmap.NodeInfo
+	nodeV2, err := cfg.nodeState.LocalNodeInfo()
+	if err != nil {
+		return nil, err
+	}
+	err = node.ReadFromV2(*nodeV2)
+	if err != nil {
+		return nil, err
+	}
+	b := make([]byte, uint16Bytes)
+	for i := range unsortedVector {
+		for _, m := range cfg.metrics {
+			metrics[i] = append(metrics[i], m.CalculateValue(&node, &unsortedVector[i])...)
+		}
+		binary.LittleEndian.PutUint16(b, uint16(i))
+		metrics[i] = append(metrics[i], b...)
+	}
+	count := len(metrics[0]) - uint16Bytes
+	slices.SortFunc(metrics, func(a, b []byte) int {
+		return bytes.Compare(a[:count], b[:count])
+	})
+	sortedVector := make([]netmap.NodeInfo, len(unsortedVector))
+	for i := range unsortedVector {
+		index := binary.LittleEndian.Uint16(metrics[i][count:])
+		sortedVector[i] = unsortedVector[index]
+	}
+	return sortedVector, nil
+}
+
 // Node is a descriptor of storage node with information required for intra-container communication.
 type Node struct {
 	addresses network.AddressGroup
@@ -322,3 +388,17 @@ func WithCopyNumbers(v []uint32) Option {
 		c.copyNumbers = v
 	}
 }
+
+// WithPriorityMetrics use provided priority metrics to sort nodes.
+func WithPriorityMetrics(m []Metric) Option {
+	return func(c *cfg) {
+		c.metrics = m
+	}
+}
+
+// WithNodeState provide state of the current node.
+func WithNodeState(s NodeState) Option {
+	return func(c *cfg) {
+		c.nodeState = s
+	}
+}

pkg/services/object_manager/placement/traverser_test.go

@@ -4,6 +4,7 @@ import (
 	"strconv"
 	"testing"
 
+	netmapAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
 	netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
@@ -22,7 +23,9 @@ func (b testBuilder) BuildPlacement(cid.ID, *oid.ID, netmap.PlacementPolicy) ([]
 }
 
 func testNode(v uint32) (n netmap.NodeInfo) {
-	n.SetNetworkEndpoints("/ip4/0.0.0.0/tcp/" + strconv.Itoa(int(v)))
+	ip := "/ip4/0.0.0.0/tcp/" + strconv.Itoa(int(v))
+	n.SetNetworkEndpoints(ip)
+	n.SetPublicKey([]byte(ip))
 
 	return n
 }
@@ -134,7 +137,7 @@ func TestTraverserObjectScenarios(t *testing.T) {
 		err = n.FromIterator(netmapcore.Node(nodes[1][0]))
 		require.NoError(t, err)
 
-		require.Equal(t, []Node{{addresses: n}}, tr.Next())
+		require.Equal(t, []Node{{addresses: n, key: []byte("/ip4/0.0.0.0/tcp/5")}}, tr.Next())
 	})
 
 	t.Run("put scenario", func(t *testing.T) {
@@ -275,3 +278,197 @@ func TestTraverserRemValues(t *testing.T) {
 		})
 	}
 }
+
+type nodeState struct {
+	node *netmapAPI.NodeInfo
+}
+
+func (n *nodeState) LocalNodeInfo() (*netmapAPI.NodeInfo, error) {
+	return n.node, nil
+}
+
+func TestTraverserPriorityMetrics(t *testing.T) {
+	t.Run("one rep one metric", func(t *testing.T) {
+		selectors := []int{4}
+		replicas := []int{3}
+
+		nodes, cnr := testPlacement(selectors, replicas)
+
+		nodes[0][0].SetAttribute("ClusterName", "A")
+		nodes[0][1].SetAttribute("ClusterName", "A")
+		nodes[0][2].SetAttribute("ClusterName", "B")
+		nodes[0][3].SetAttribute("ClusterName", "B")
+
+		sdkNode := testNode(5)
+		sdkNode.SetAttribute("ClusterName", "B")
+		nodeAPI := &netmapAPI.NodeInfo{}
+		sdkNode.WriteToV2(nodeAPI)
+
+		nodesCopy := copyVectors(nodes)
+
+		m := []Metric{NewAttributeMetric("$attribute:ClusterName")}
+
+		tr, err := NewTraverser(
+			ForContainer(cnr),
+			UseBuilder(&testBuilder{
+				vectors: nodesCopy,
+			}),
+			WithoutSuccessTracking(),
+			WithPriorityMetrics(m),
+			WithNodeState(&nodeState{
+				node: nodeAPI,
+			}),
+		)
+		require.NoError(t, err)
+
+		next := tr.Next()
+		require.NotNil(t, next)
+		require.Equal(t, 3, len(next))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[1].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[2].PublicKey()))
+
+		next = tr.Next()
+		require.Equal(t, 1, len(next))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[0].PublicKey()))
+
+		next = tr.Next()
+		require.Nil(t, next)
+	})
+
+	t.Run("two reps two metrics", func(t *testing.T) {
+		selectors := []int{3, 3}
+		replicas := []int{2, 2}
+
+		nodes, cnr := testPlacement(selectors, replicas)
+
+		nodes[0][0].SetAttribute("ClusterName", "A")
+		nodes[0][0].SetAttribute("UN-LOCODE", "RU LED")
+
+		nodes[0][1].SetAttribute("ClusterName", "A")
+		nodes[0][1].SetAttribute("UN-LOCODE", "FI HEL")
+
+		nodes[0][2].SetAttribute("ClusterName", "A")
+		nodes[0][2].SetAttribute("UN-LOCODE", "RU LED")
+
+		nodes[1][0].SetAttribute("ClusterName", "B")
+		nodes[1][0].SetAttribute("UN-LOCODE", "RU MOW")
+
+		nodes[1][1].SetAttribute("ClusterName", "B")
+		nodes[1][1].SetAttribute("UN-LOCODE", "RU DME")
+
+		nodes[1][2].SetAttribute("ClusterName", "B")
+		nodes[1][2].SetAttribute("UN-LOCODE", "RU MOW")
+
+		sdkNode := testNode(9)
+		sdkNode.SetAttribute("ClusterName", "B")
+		sdkNode.SetAttribute("UN-LOCODE", "RU DME")
+		nodeAPI := &netmapAPI.NodeInfo{}
+		sdkNode.WriteToV2(nodeAPI)
+
+		nodesCopy := copyVectors(nodes)
+
+		m := []Metric{
+			NewAttributeMetric("$attribute:ClusterName"),
+			NewAttributeMetric("$attribute:UN-LOCODE"),
+		}
+
+		tr, err := NewTraverser(
+			ForContainer(cnr),
+			UseBuilder(&testBuilder{
+				vectors: nodesCopy,
+			}),
+			WithoutSuccessTracking(),
+			WithPriorityMetrics(m),
+			WithNodeState(&nodeState{
+				node: nodeAPI,
+			}),
+		)
+		require.NoError(t, err)
+
+		next := tr.Next()
+		require.Equal(t, 4, len(next))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/4", string(next[0].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[1].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[2].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[3].PublicKey()))
+
+		next = tr.Next()
+		require.Equal(t, 2, len(next))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/5", string(next[1].PublicKey()))
+
+		next = tr.Next()
+		require.Nil(t, next)
+
+		sdkNode.SetAttribute("ClusterName", "B")
+		sdkNode.SetAttribute("UN-LOCODE", "RU MOW")
+		nodeAPI = &netmapAPI.NodeInfo{}
+		sdkNode.WriteToV2(nodeAPI)
+
+		nodesCopy = copyVectors(nodes)
+
+		tr, err = NewTraverser(
+			ForContainer(cnr),
+			UseBuilder(&testBuilder{
+				vectors: nodesCopy,
+			}),
+			WithoutSuccessTracking(),
+			WithPriorityMetrics(m),
+			WithNodeState(&nodeState{
+				node: nodeAPI,
+			}),
+		)
+		require.NoError(t, err)
+
+		next = tr.Next()
+		require.Equal(t, 4, len(next))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[0].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/4", string(next[1].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[2].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[3].PublicKey()))
+
+		next = tr.Next()
+		require.Equal(t, 2, len(next))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/5", string(next[1].PublicKey()))
+
+		next = tr.Next()
+		require.Nil(t, next)
+
+		sdkNode.SetAttribute("ClusterName", "A")
+		sdkNode.SetAttribute("UN-LOCODE", "RU LED")
+		nodeAPI = &netmapAPI.NodeInfo{}
+		sdkNode.WriteToV2(nodeAPI)
+
+		nodesCopy = copyVectors(nodes)
+
+		tr, err = NewTraverser(
+			ForContainer(cnr),
+			UseBuilder(&testBuilder{
+				vectors: nodesCopy,
+			}),
+			WithoutSuccessTracking(),
+			WithPriorityMetrics(m),
+			WithNodeState(&nodeState{
+				node: nodeAPI,
+			}),
+		)
+		require.NoError(t, err)
+
+		next = tr.Next()
+		require.Equal(t, 4, len(next))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[0].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[1].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[2].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/4", string(next[3].PublicKey()))
+
+		next = tr.Next()
+		require.Equal(t, 2, len(next))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey()))
+		require.Equal(t, "/ip4/0.0.0.0/tcp/5", string(next[1].PublicKey()))
+
+		next = tr.Next()
+		require.Nil(t, next)
+	})
+}