CHANGELOG.md

@@ -15,6 +15,7 @@ This document outlines major changes between releases.
 - Fix parsing signed headers in presigned urls (#182)
 
 ### Added
+- Add a metric with addresses of nodes of the same and highest priority that are currently healthy (#51)
 - Support dump metrics descriptions (#80)
 - Add `copies_numbers` section to `placement_policy` in config file and support vectors of copies numbers (#70, #101)
 - Support impersonate bearer token (#81, #105)

api/auth/presign_test.go

@@ -36,7 +36,7 @@ func (m credentialsMock) addBox(addr oid.Address, box *accessbox.Box) {
 func (m credentialsMock) GetBox(_ context.Context, addr oid.Address) (*accessbox.Box, error) {
 	box, ok := m.boxes[addr.String()]
 	if !ok {
-		return nil, apistatus.ObjectNotFound{}
+		return nil, &apistatus.ObjectNotFound{}
 	}
 
 	return box, nil

api/handler/delete.go

@@ -150,8 +150,7 @@ func isErrObjectLocked(err error) bool {
 	switch err.(type) {
 	default:
 		return strings.Contains(err.Error(), "object is locked")
-	case apistatus.ObjectLocked,
+	case *apistatus.ObjectLocked:
-		*apistatus.ObjectLocked:
 		return true
 	}
 }

api/handler/delete_test.go

@@ -28,7 +28,7 @@ func TestDeleteBucketOnAlreadyRemovedError(t *testing.T) {
 	putObject(t, hc, bktName, objName)
 
 	addr := getAddressOfLastVersion(hc, bktInfo, objName)
-	hc.tp.SetObjectError(addr, apistatus.ObjectAlreadyRemoved{})
+	hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
 
 	deleteObjects(t, hc, bktName, [][2]string{{objName, emptyVersion}})
 
@@ -73,7 +73,7 @@ func TestDeleteBucketOnNotFoundError(t *testing.T) {
 	var addr oid.Address
 	addr.SetContainer(bktInfo.CID)
 	addr.SetObject(nodeVersion.OID)
-	hc.tp.SetObjectError(addr, apistatus.ObjectNotFound{})
+	hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
 
 	deleteObjects(t, hc, bktName, [][2]string{{objName, emptyVersion}})
 

api/handler/get_test.go

@@ -192,8 +192,8 @@ func TestGetObject(t *testing.T) {
 	checkFound(hc.t, hc, bktName, objName, emptyVersion)
 
 	addr := getAddressOfLastVersion(hc, bktInfo, objName)
-	hc.tp.SetObjectError(addr, apistatus.ObjectNotFound{})
+	hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
-	hc.tp.SetObjectError(objInfo.Address(), apistatus.ObjectNotFound{})
+	hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
 
 	getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), s3errors.ErrNoSuchVersion)
 	getObjectAssertS3Error(hc, bktName, objName, emptyVersion, s3errors.ErrNoSuchKey)

api/handler/head_test.go

@@ -120,8 +120,8 @@ func TestHeadObject(t *testing.T) {
 	checkFound(hc.t, hc, bktName, objName, emptyVersion)
 
 	addr := getAddressOfLastVersion(hc, bktInfo, objName)
-	hc.tp.SetObjectError(addr, apistatus.ObjectNotFound{})
+	hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
-	hc.tp.SetObjectError(objInfo.Address(), apistatus.ObjectNotFound{})
+	hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
 
 	headObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), s3errors.ErrNoSuchVersion)
 	headObjectAssertS3Error(hc, bktName, objName, emptyVersion, s3errors.ErrNoSuchKey)

api/layer/frostfs_mock.go

@@ -198,7 +198,7 @@ func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*Objec
 		}, nil
 	}
 
-	return nil, fmt.Errorf("%w: %s", apistatus.ObjectNotFound{}, addr)
+	return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, addr)
 }
 
 func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID, error) {

go.mod

@@ -5,7 +5,7 @@ go 1.19
 require (
 	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44
 	git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6
-	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6
+	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e
 	github.com/aws/aws-sdk-go v1.44.6
 	github.com/bluele/gcache v0.0.2
 	github.com/go-chi/chi/v5 v5.0.8

go.sum

@@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6 h1:u6lzNotV6MEMNEG/XeS7g+FjPrrf+j4gnOHtvun2KJc=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e h1:+78sZdTFeK2t+2qJ4Tmm5CItUjfcF4/nFMum3JnZ6PQ=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6/go.mod h1:LI2GOj0pEx0jYTjB3QHja2PNhQFYL2pCm71RAFwDv0M=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
 git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA=

internal/frostfs/errors/errors.go

@@ -25,8 +25,6 @@ func IsErrObjectAccessDenied(err error) (string, bool) {
 	switch err := err.(type) {
 	default:
 		return "", false
-	case apistatus.ObjectAccessDenied:
-		return err.Reason(), true
 	case *apistatus.ObjectAccessDenied:
 		return err.Reason(), true
 	}

metrics/desc.go

@@ -48,6 +48,14 @@ var appMetricsDesc = map[string]map[string]Description{
 			Help:           "Average request duration (in milliseconds) for specific method on node in pool",
 			VariableLabels: []string{"node", "method"},
 		},
+		currentNodesMetric: Description{
+			Type:           dto.MetricType_GAUGE,
+			Namespace:      namespace,
+			Subsystem:      poolSubsystem,
+			Name:           currentNodesMetric,
+			Help:           "Addresses of nodes of the same and highest priority that are currently healthy",
+			VariableLabels: []string{"address"},
+		},
 	},
 	billingSubsystem: {
 		userRequestsMetric: Description{

metrics/pool.go

@@ -15,6 +15,7 @@ const (
 	overallNodeRequestsMetric = "overall_node_requests"
 	currentErrorMetric        = "current_errors"
 	avgRequestDurationMetric  = "avg_request_duration"
+	currentNodesMetric        = "current_nodes"
 )
 
 const (
@@ -42,6 +43,7 @@ type poolMetricsCollector struct {
 	overallNodeRequests *prometheus.GaugeVec
 	currentErrors       *prometheus.GaugeVec
 	requestDuration     *prometheus.GaugeVec
+	currentNodes        *prometheus.GaugeVec
 }
 
 func newPoolMetricsCollector(scraper StatisticScraper) *poolMetricsCollector {
@@ -52,6 +54,7 @@ func newPoolMetricsCollector(scraper StatisticScraper) *poolMetricsCollector {
 		overallNodeRequests: mustNewGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]),
 		currentErrors:       mustNewGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]),
 		requestDuration:     mustNewGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]),
+		currentNodes:        mustNewGaugeVec(appMetricsDesc[poolSubsystem][currentNodesMetric]),
 	}
 }
 
@@ -62,6 +65,7 @@ func (m *poolMetricsCollector) Collect(ch chan<- prometheus.Metric) {
 	m.overallNodeRequests.Collect(ch)
 	m.currentErrors.Collect(ch)
 	m.requestDuration.Collect(ch)
+	m.currentNodes.Collect(ch)
 }
 
 func (m *poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) {
@@ -70,6 +74,7 @@ func (m *poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) {
 	m.overallNodeRequests.Describe(descs)
 	m.currentErrors.Describe(descs)
 	m.requestDuration.Describe(descs)
+	m.currentNodes.Describe(descs)
 }
 
 func (m *poolMetricsCollector) updateStatistic() {
@@ -79,6 +84,7 @@ func (m *poolMetricsCollector) updateStatistic() {
 	m.overallNodeRequests.Reset()
 	m.currentErrors.Reset()
 	m.requestDuration.Reset()
+	m.currentNodes.Reset()
 
 	for _, node := range stat.Nodes() {
 		m.overallNodeErrors.WithLabelValues(node.Address()).Set(float64(node.OverallErrors()))
@@ -88,6 +94,10 @@ func (m *poolMetricsCollector) updateStatistic() {
 		m.updateRequestsDuration(node)
 	}
 
+	for _, addr := range stat.CurrentNodes() {
+		m.currentNodes.WithLabelValues(addr).Set(1)
+	}
+
 	m.overallErrors.Set(float64(stat.OverallErrors()))
 }