TrueCloudLab/frostfs-s3-gw
17
3
Fork 20
Code Issues 29 Pull requests 8 Projects Releases 30 Packages 1 Activity Actions

Add node addresses metric #186

Merged
dkirillov merged 2 commits from ironbee/frostfs-s3-gw:add-node-addresses-metric into master 2024-09-04 19:51:12 +00:00
Conversation 0 Commits 2 Files changed 12 +31 -15
12 changed files with 31 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -15,6 +15,7 @@ This document outlines major changes between releases.
- Fix parsing signed headers in presigned urls (#182)
### Added
- Add a metric with addresses of nodes of the same and highest priority that are currently healthy (#51)
- Support dump metrics descriptions (#80)
- Add `copies_numbers` section to `placement_policy` in config file and support vectors of copies numbers (#70, #101)
- Support impersonate bearer token (#81, #105)

2
api/auth/presign_test.go
View file

@ -36,7 +36,7 @@ func (m credentialsMock) addBox(addr oid.Address, box *accessbox.Box) {
func (m credentialsMock) GetBox(_ context.Context, addr oid.Address) (*accessbox.Box, error) {
box, ok := m.boxes[addr.String()]
if !ok {
return nil, apistatus.ObjectNotFound{}
return nil, &apistatus.ObjectNotFound{}
}
return box, nil

3
api/handler/delete.go
View file

@ -150,8 +150,7 @@ func isErrObjectLocked(err error) bool {
switch err.(type) {
default:
return strings.Contains(err.Error(), "object is locked")
case apistatus.ObjectLocked,
*apistatus.ObjectLocked:
case *apistatus.ObjectLocked:
return true
}
}

4
api/handler/delete_test.go
View file

@ -28,7 +28,7 @@ func TestDeleteBucketOnAlreadyRemovedError(t *testing.T) {
putObject(t, hc, bktName, objName)
addr := getAddressOfLastVersion(hc, bktInfo, objName)
hc.tp.SetObjectError(addr, apistatus.ObjectAlreadyRemoved{})
hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
deleteObjects(t, hc, bktName, [][2]string{{objName, emptyVersion}})
@ -73,7 +73,7 @@ func TestDeleteBucketOnNotFoundError(t *testing.T) {
var addr oid.Address
addr.SetContainer(bktInfo.CID)
addr.SetObject(nodeVersion.OID)
hc.tp.SetObjectError(addr, apistatus.ObjectNotFound{})
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
deleteObjects(t, hc, bktName, [][2]string{{objName, emptyVersion}})

4
api/handler/get_test.go
View file

@ -192,8 +192,8 @@ func TestGetObject(t *testing.T) {
checkFound(hc.t, hc, bktName, objName, emptyVersion)
addr := getAddressOfLastVersion(hc, bktInfo, objName)
hc.tp.SetObjectError(addr, apistatus.ObjectNotFound{})
hc.tp.SetObjectError(objInfo.Address(), apistatus.ObjectNotFound{})
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), s3errors.ErrNoSuchVersion)
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, s3errors.ErrNoSuchKey)

4
api/handler/head_test.go
View file

@ -120,8 +120,8 @@ func TestHeadObject(t *testing.T) {
checkFound(hc.t, hc, bktName, objName, emptyVersion)
addr := getAddressOfLastVersion(hc, bktInfo, objName)
hc.tp.SetObjectError(addr, apistatus.ObjectNotFound{})
hc.tp.SetObjectError(objInfo.Address(), apistatus.ObjectNotFound{})
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
headObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), s3errors.ErrNoSuchVersion)
headObjectAssertS3Error(hc, bktName, objName, emptyVersion, s3errors.ErrNoSuchKey)

2
api/layer/frostfs_mock.go
View file

@ -198,7 +198,7 @@ func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*Objec
}, nil
}
return nil, fmt.Errorf("%w: %s", apistatus.ObjectNotFound{}, addr)
return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, addr)
}
func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID, error) {

2
go.mod
View file

@ -5,7 +5,7 @@ go 1.19
require (
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e
github.com/aws/aws-sdk-go v1.44.6
github.com/bluele/gcache v0.0.2
github.com/go-chi/chi/v5 v5.0.8

4
go.sum
View file

@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6 h1:u6lzNotV6MEMNEG/XeS7g+FjPrrf+j4gnOHtvun2KJc=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6/go.mod h1:LI2GOj0pEx0jYTjB3QHja2PNhQFYL2pCm71RAFwDv0M=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e h1:+78sZdTFeK2t+2qJ4Tmm5CItUjfcF4/nFMum3JnZ6PQ=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA=

2
internal/frostfs/errors/errors.go
View file

@ -25,8 +25,6 @@ func IsErrObjectAccessDenied(err error) (string, bool) {
switch err := err.(type) {
default:
return "", false
case apistatus.ObjectAccessDenied:
return err.Reason(), true
case *apistatus.ObjectAccessDenied:
return err.Reason(), true
}

8
metrics/desc.go
View file

@ -48,6 +48,14 @@ var appMetricsDesc = map[string]map[string]Description{
Help: "Average request duration (in milliseconds) for specific method on node in pool",
VariableLabels: []string{"node", "method"},
},
currentNodesMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: poolSubsystem,
Name: currentNodesMetric,
Help: "Addresses of nodes of the same and highest priority that are currently healthy",
VariableLabels: []string{"address"},
},
},
billingSubsystem: {
userRequestsMetric: Description{

10
metrics/pool.go
View file

@ -15,6 +15,7 @@ const (
overallNodeRequestsMetric = "overall_node_requests"
currentErrorMetric = "current_errors"
avgRequestDurationMetric = "avg_request_duration"
currentNodesMetric = "current_nodes"
)
const (
@ -42,6 +43,7 @@ type poolMetricsCollector struct {
overallNodeRequests *prometheus.GaugeVec
currentErrors *prometheus.GaugeVec
requestDuration *prometheus.GaugeVec
currentNodes *prometheus.GaugeVec
}
func newPoolMetricsCollector(scraper StatisticScraper) *poolMetricsCollector {
@ -52,6 +54,7 @@ func newPoolMetricsCollector(scraper StatisticScraper) *poolMetricsCollector {
overallNodeRequests: mustNewGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]),
currentErrors: mustNewGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]),
requestDuration: mustNewGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]),
currentNodes: mustNewGaugeVec(appMetricsDesc[poolSubsystem][currentNodesMetric]),
}
}
@ -62,6 +65,7 @@ func (m *poolMetricsCollector) Collect(ch chan<- prometheus.Metric) {
m.overallNodeRequests.Collect(ch)
m.currentErrors.Collect(ch)
m.requestDuration.Collect(ch)
m.currentNodes.Collect(ch)
}
func (m *poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) {
@ -70,6 +74,7 @@ func (m *poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) {
m.overallNodeRequests.Describe(descs)
m.currentErrors.Describe(descs)
m.requestDuration.Describe(descs)
m.currentNodes.Describe(descs)
}
func (m *poolMetricsCollector) updateStatistic() {
@ -79,6 +84,7 @@ func (m *poolMetricsCollector) updateStatistic() {
m.overallNodeRequests.Reset()
m.currentErrors.Reset()
m.requestDuration.Reset()
m.currentNodes.Reset()
for _, node := range stat.Nodes() {
m.overallNodeErrors.WithLabelValues(node.Address()).Set(float64(node.OverallErrors()))
@ -88,6 +94,10 @@ func (m *poolMetricsCollector) updateStatistic() {
m.updateRequestsDuration(node)
}
for _, addr := range stat.CurrentNodes() {
m.currentNodes.WithLabelValues(addr).Set(1)
}
m.overallErrors.Set(float64(stat.OverallErrors()))
}