Release v0.28.1

Signed-off-by: Alex Vanin <a.vanin@yadro.com>

.gitignore

@@ -18,4 +18,12 @@ coverage.txt
 coverage.html
 
 # debhelpers
-**/.debhelper
+**/*debhelper*
+
+# debian package build files
+debian/files
+debian/changelog
+debian/*.log
+debian/*.substvars
+debian/frostfs-http-gw/
+

CHANGELOG.md

@@ -4,20 +4,39 @@ This document outlines major changes between releases.
 
 ## [Unreleased]
 
+## [0.28.1] - 2024-01-24
+
+### Added
+- Tree pool traversal limit (#92)
+
+### Update from 0.28.0
+See new `frostfs.tree_pool_max_attempts` config parameter.
+
+
+## [0.28.0] - Academy of Sciences - 2023-12-07
+
 ### Fixed
 - `grpc` schemas in tree configuration (#62)
+- `GetSubTree` failures (#67)
+- Debian packaging (#69, #90)
+- Get latest version of tree node (#85)
 
 ### Added
 - Support dump metrics descriptions (#29)
 - Support impersonate bearer token (#40, #45)
 - Tracing support (#20, #44, #60)
 - Object name resolving with tree service (#30)
+- Metrics for current endpoint status (#77)
+- Soft memory limit with `runtime.soft_memory_limit` (#72)
+- Add selection of the node of the latest version of the object (#85)
 
 ### Changed
 - Update prometheus to v1.15.0 (#35)
 - Update go version to 1.19 (#50)
 - Finish rebranding (#2)
 - Use gate key to form object owner (#66)
+- Move log messages to constants (#36)
+- Uploader and downloader refactor (#73)
 
 ### Removed
 - Drop `tree.service` param (now endpoints from `peers` section are used) (#59)
@@ -61,4 +80,6 @@ This project is a fork of [NeoFS HTTP Gateway](https://github.com/nspcc-dev/neof
 To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs-http-gw/blob/master/CHANGELOG.md.
 
 [0.27.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/72734ab4...v0.27.0
-[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.27.0...master
+[0.28.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.27.0...v0.28.0
+[0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.0...v0.28.1
+[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.1...master

Makefile

@@ -153,6 +153,7 @@ clean:
 
 # Package for Debian
 debpackage:
+	cp debian/changelog.init debian/changelog
 	dch --package frostfs-http-gw \
 			--controlmaint \
 			--newversion $(PKG_VERSION) \

VERSION

@@ -1 +1 @@
-v0.27.0
+v0.28.1

cmd/http-gw/app.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/signal"
 	"runtime/debug"
+	"strconv"
 	"sync"
 	"syscall"
 	"time"
@@ -497,6 +498,8 @@ func (a *app) tracer(h fasthttp.RequestHandler) fasthttp.RequestHandler {
 			span.End()
 		}()
 
+		appCtx = treepool.SetRequestID(appCtx, strconv.FormatUint(req.ID(), 10))
+
 		utils.SetContextToRequest(appCtx, req)
 		h(req)
 	}

cmd/http-gw/settings.go

@@ -96,6 +96,10 @@ const (
 	// Runtime.
 	cfgSoftMemoryLimit = "runtime.soft_memory_limit"
 
+	// Configuration of parameters of requests to FrostFS.
+	// Sets max attempt to make successful tree request.
+	cfgTreePoolMaxAttempts = "frostfs.tree_pool_max_attempts"
+
 	// Command line args.
 	cmdHelp          = "help"
 	cmdVersion       = "version"
@@ -461,6 +465,8 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.
 	prm.SetLogger(logger)
 	prmTree.SetLogger(logger)
 
+	prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgTreePoolMaxAttempts))
+
 	var apiGRPCDialOpts []grpc.DialOption
 	var treeGRPCDialOpts []grpc.DialOption
 	if cfg.GetBool(cfgTracingEnabled) {

config/config.env

@@ -98,3 +98,7 @@ HTTP_GW_TRACING_ENDPOINT="localhost:4317"
 HTTP_GW_TRACING_EXPORTER="otlp_grpc"
 
 HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824
+
+# Max attempt to make successful tree request.
+# default value is 0 that means the number of attempts equals to number of nodes in pool.
+HTTP_GW_FROSTFS_TREE_POOL_MAX_ATTEMPTS=0

config/config.yaml

@@ -104,3 +104,8 @@ zip:
 
 runtime:
   soft_memory_limit: 1gb
+
+frostfs:
+  # Max attempt to make successful tree request.
+  # default value is 0 that means the number of attempts equals to number of nodes in pool.
+  tree_pool_max_attempts: 0

docs/gate-configuration.md

@@ -54,6 +54,7 @@ $ cat http.log
 | `prometheus`    | [Prometheus configuration](#prometheus-section)       |
 | `tracing`       | [Tracing configuration](#tracing-section)             |
 | `runtime`       | [Runtime configuration](#runtime-section)             |
+| `frostfs`       | [Parameters of requests to FrostFS](#frostfs-section) |
 
 
 # General section
@@ -269,3 +270,16 @@ runtime:
 | Parameter           | Type   | SIGHUP reload | Default value | Description                                                                                                                                                                  |
 |---------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `soft_memory_limit` | `size` | yes           | maxint64      | Soft memory limit for the runtime. Zero or no value stands for no limit. If `GOMEMLIMIT` environment variable is set, the value from the configuration file will be ignored. |
+
+# `frostfs` section
+
+Contains parameters of requests to FrostFS.
+
+```yaml
+frostfs:
+  tree_pool_max_attempts: 0
+```
+
+| Parameter                | Type     | SIGHUP reload | Default value | Description                                                                                                               |
+|--------------------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------|
+| `tree_pool_max_attempts` | `uint32` | no            | `0`           | Sets max attempt to make successful tree request. Value 0 means the number of attempts equals to number of nodes in pool. |

go.mod

@@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw
 go 1.20
 
 require (
-	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44
+	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4
 	git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6
-	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6
+	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939
 	github.com/fasthttp/router v1.4.1
 	github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc
 	github.com/prometheus/client_golang v1.15.1

go.sum

@@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 h1:v6JqBD/VzZx3QSxbaXnUwnnJ1KEYheU4LzLGr3IhsAE=
-git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI=
+git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4 h1:wjLfZ3WCt7qNGsQv+Jl0TXnmtg0uVk/jToKPFTBc/jo=
+git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI=
 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M=
 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o=
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6 h1:u6lzNotV6MEMNEG/XeS7g+FjPrrf+j4gnOHtvun2KJc=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6/go.mod h1:LI2GOj0pEx0jYTjB3QHja2PNhQFYL2pCm71RAFwDv0M=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939 h1:jZEepi9yWmqrWgLRQcHQu4YPJaudmd7d2AEhpmM3m4U=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
 git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA=

internal/handler/download.go

@@ -64,8 +64,9 @@ func (h *Handler) search(ctx context.Context, cid *cid.ID, key, val string, op o
 }
 
 func (h *Handler) getContainer(ctx context.Context, cnrID cid.ID) (container.Container, error) {
-	var prm pool.PrmContainerGet
-	prm.SetContainerID(cnrID)
+	prm := pool.PrmContainerGet{
+		ContainerID: cnrID,
+	}
 
 	return h.pool.GetContainer(ctx, prm)
 }

tree/tree.go

@@ -73,6 +73,7 @@ type Meta interface {
 
 type NodeResponse interface {
 	GetMeta() []Meta
+	GetTimestamp() uint64
 }
 
 func newTreeNode(nodeInfo NodeResponse) (*treeNode, error) {
@@ -135,7 +136,7 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s
 		TreeID:     versionTree,
 		Path:       path,
 		Meta:       meta,
-		LatestOnly: true,
+		LatestOnly: false,
 		AllAttrs:   false,
 	}
 	nodes, err := c.service.GetNodes(ctx, p)
@@ -143,11 +144,43 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s
 		return nil, err
 	}
 
-	if len(nodes) == 0 {
+	latestNode, err := getLatestNode(nodes)
+	if err != nil {
+		return nil, err
+	}
+
+	return newNodeVersion(latestNode)
+}
+
+func getLatestNode(nodes []NodeResponse) (NodeResponse, error) {
+	var (
+		maxCreationTime uint64
+		targetIndexNode = -1
+	)
+
+	for i, node := range nodes {
+		currentCreationTime := node.GetTimestamp()
+		if checkExistOID(node.GetMeta()) && currentCreationTime > maxCreationTime {
+			maxCreationTime = currentCreationTime
+			targetIndexNode = i
+		}
+	}
+
+	if targetIndexNode == -1 {
 		return nil, layer.ErrNodeNotFound
 	}
 
-	return newNodeVersion(nodes[0])
+	return nodes[targetIndexNode], nil
+}
+
+func checkExistOID(meta []Meta) bool {
+	for _, kv := range meta {
+		if kv.GetKey() == "OID" {
+			return true
+		}
+	}
+
+	return false
 }
 
 // pathFromName splits name by '/'.

tree/tree_test.go

@@ -0,0 +1,143 @@
+package tree
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type nodeMeta struct {
+	key   string
+	value []byte
+}
+
+func (m nodeMeta) GetKey() string {
+	return m.key
+}
+
+func (m nodeMeta) GetValue() []byte {
+	return m.value
+}
+
+type nodeResponse struct {
+	meta      []nodeMeta
+	timestamp uint64
+}
+
+func (n nodeResponse) GetTimestamp() uint64 {
+	return n.timestamp
+}
+
+func (n nodeResponse) GetMeta() []Meta {
+	res := make([]Meta, len(n.meta))
+	for i, value := range n.meta {
+		res[i] = value
+	}
+	return res
+}
+
+func TestGetLatestNode(t *testing.T) {
+	for _, tc := range []struct {
+		name        string
+		nodes       []NodeResponse
+		exceptedOID string
+		error       bool
+	}{
+		{
+			name:  "empty",
+			nodes: []NodeResponse{},
+			error: true,
+		},
+		{
+			name: "one node of the object version",
+			nodes: []NodeResponse{
+				nodeResponse{
+					timestamp: 1,
+					meta: []nodeMeta{
+						{
+							key:   oidKV,
+							value: []byte("oid1"),
+						},
+					},
+				},
+			},
+			exceptedOID: "oid1",
+		},
+		{
+			name: "one node of the object version and one node of the secondary object",
+			nodes: []NodeResponse{
+				nodeResponse{
+					timestamp: 3,
+					meta:      []nodeMeta{},
+				},
+				nodeResponse{
+					timestamp: 1,
+					meta: []nodeMeta{
+						{
+							key:   oidKV,
+							value: []byte("oid1"),
+						},
+					},
+				},
+			},
+			exceptedOID: "oid1",
+		},
+		{
+			name: "all nodes represent a secondary object",
+			nodes: []NodeResponse{
+				nodeResponse{
+					timestamp: 3,
+					meta:      []nodeMeta{},
+				},
+				nodeResponse{
+					timestamp: 5,
+					meta:      []nodeMeta{},
+				},
+			},
+			error: true,
+		},
+		{
+			name: "several nodes of different types and with different timestamp",
+			nodes: []NodeResponse{
+				nodeResponse{
+					timestamp: 1,
+					meta: []nodeMeta{
+						{
+							key:   oidKV,
+							value: []byte("oid1"),
+						},
+					},
+				},
+				nodeResponse{
+					timestamp: 3,
+					meta:      []nodeMeta{},
+				},
+				nodeResponse{
+					timestamp: 4,
+					meta: []nodeMeta{
+						{
+							key:   oidKV,
+							value: []byte("oid2"),
+						},
+					},
+				},
+				nodeResponse{
+					timestamp: 6,
+					meta:      []nodeMeta{},
+				},
+			},
+			exceptedOID: "oid2",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			actualNode, err := getLatestNode(tc.nodes)
+			if tc.error {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.Equal(t, tc.exceptedOID, string(actualNode.GetMeta()[0].GetValue()))
+		})
+	}
+}