Small perfomance optimizations, cleanup #236

Merged
fyrchik merged 7 commits from fyrchik/frostfs-sdk-go:sort-faster into master 2024-09-04 19:51:16 +00:00
12 changed files with 67 additions and 76 deletions

View file

@ -13,7 +13,7 @@ jobs:
- name: Setup Go - name: Setup Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '1.21' go-version: '1.22'
- name: Run commit format checker - name: Run commit format checker
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3

View file

@ -11,7 +11,7 @@ jobs:
- name: Set up Go - name: Set up Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '1.21' go-version: '1.22'
cache: true cache: true
- name: Install linters - name: Install linters
@ -25,7 +25,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
go_versions: [ '1.20', '1.21' ] go_versions: [ '1.21', '1.22' ]
fail-fast: false fail-fast: false
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3

43
go.mod
View file

@ -1,6 +1,6 @@
module git.frostfs.info/TrueCloudLab/frostfs-sdk-go module git.frostfs.info/TrueCloudLab/frostfs-sdk-go
go 1.20 go 1.21
require ( require (
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3
@ -9,41 +9,40 @@ require (
git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/hrw v1.2.1
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/tzhash v1.8.0
github.com/antlr4-go/antlr/v4 v4.13.0 github.com/antlr4-go/antlr/v4 v4.13.0
github.com/google/uuid v1.3.0 github.com/google/uuid v1.6.0
github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/klauspost/reedsolomon v1.12.1 github.com/klauspost/reedsolomon v1.12.1
github.com/mr-tron/base58 v1.2.0 github.com/mr-tron/base58 v1.2.0
github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/nspcc-dev/neo-go v0.106.2
github.com/stretchr/testify v1.8.3 github.com/stretchr/testify v1.9.0
go.uber.org/zap v1.24.0 go.uber.org/zap v1.27.0
google.golang.org/grpc v1.55.0 google.golang.org/grpc v1.62.0
google.golang.org/protobuf v1.33.0 google.golang.org/protobuf v1.33.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
) )
require ( require (
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect github.com/golang/protobuf v1.5.3 // indirect
github.com/gorilla/websocket v1.5.0 // indirect github.com/golang/snappy v0.0.1 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.8.1 // indirect github.com/rogpeppe/go-internal v1.8.1 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect
github.com/twmb/murmur3 v1.1.8 // indirect github.com/twmb/murmur3 v1.1.8 // indirect
go.uber.org/atomic v1.10.0 // indirect go.etcd.io/bbolt v1.3.9 // indirect
go.uber.org/goleak v1.2.1 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.9.0 // indirect golang.org/x/crypto v0.21.0 // indirect
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
golang.org/x/net v0.10.0 // indirect golang.org/x/net v0.23.0 // indirect
golang.org/x/sync v0.2.0 // indirect golang.org/x/sync v0.6.0 // indirect
golang.org/x/sys v0.8.0 // indirect golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.9.0 // indirect golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect
) )

BIN
go.sum

Binary file not shown.

View file

@ -1,8 +1,6 @@
package netmap package netmap
import ( import "slices"
"sort"
)
type ( type (
// aggregator can calculate some value across all netmap // aggregator can calculate some value across all netmap
@ -28,7 +26,6 @@ type (
} }
meanIQRAgg struct { meanIQRAgg struct {
k float64
arr []float64 arr []float64
} }
@ -74,12 +71,6 @@ func newMinAgg() aggregator {
return new(minAgg) return new(minAgg)
} }
// newMeanIQRAgg returns an aggregator which
// computes mean value of values from IQR interval.
func newMeanIQRAgg() aggregator {
return new(meanIQRAgg)
}
// newReverseMinNorm returns a normalizer which // newReverseMinNorm returns a normalizer which
// normalize values in range of 0.0 to 1.0 to a minimum value. // normalize values in range of 0.0 to 1.0 to a minimum value.
func newReverseMinNorm(min float64) normalizer { func newReverseMinNorm(min float64) normalizer {
@ -118,6 +109,10 @@ func (a *minAgg) Compute() float64 {
return a.min return a.min
} }
func (a *meanIQRAgg) clear() {
a.arr = a.arr[:0]
}
func (a *meanIQRAgg) Add(n float64) { func (a *meanIQRAgg) Add(n float64) {
a.arr = append(a.arr, n) a.arr = append(a.arr, n)
} }
@ -128,7 +123,7 @@ func (a *meanIQRAgg) Compute() float64 {
return 0 return 0
} }
sort.Slice(a.arr, func(i, j int) bool { return a.arr[i] < a.arr[j] }) slices.Sort(a.arr)
var min, max float64 var min, max float64
@ -138,8 +133,7 @@ func (a *meanIQRAgg) Compute() float64 {
min, max = a.arr[0], a.arr[l-1] min, max = a.arr[0], a.arr[l-1]
} else { } else {
start, end := l/minLn, l*3/minLn-1 start, end := l/minLn, l*3/minLn-1
iqr := a.k * (a.arr[end] - a.arr[start]) min, max = a.arr[start], a.arr[end]
min, max = a.arr[start]-iqr, a.arr[end]+iqr
} }
count := 0 count := 0

View file

@ -113,9 +113,10 @@ func (n nodes) Hash() uint64 {
return 0 return 0
} }
// weights returns slice of nodes weights W. func (n nodes) appendWeightsTo(wf weightFunc, w []float64) []float64 {
func (n nodes) weights(wf weightFunc) []float64 { if cap(w) < len(n) {
w := make([]float64, 0, len(n)) w = make([]float64, 0, len(n))
Review

The method's name is appendWeightsTo. Here you pass w, then just reallocate w but you don't save previous values. We need to fix that?

The method's name is `appendWeightsTo`. Here you pass `w`, then just reallocate `w` but you don't save previous values. We need to fix that?
}
for i := range n { for i := range n {
w = append(w, wf(n[i])) w = append(w, wf(n[i]))
} }
@ -149,10 +150,13 @@ func (m NetMap) PlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]NodeIn
wf := defaultWeightFunc(m.nodes) wf := defaultWeightFunc(m.nodes)
result := make([][]NodeInfo, len(vectors)) result := make([][]NodeInfo, len(vectors))
var ws []float64
for i := range vectors { for i := range vectors {
result[i] = make([]NodeInfo, len(vectors[i])) result[i] = make([]NodeInfo, len(vectors[i]))
copy(result[i], vectors[i]) copy(result[i], vectors[i])
hrw.SortHasherSliceByWeightValue(result[i], nodes(result[i]).weights(wf), h) ws = nodes(result[i]).appendWeightsTo(wf, ws[:0])
hrw.SortHasherSliceByWeightValue(result[i], ws, h)
} }
return result, nil return result, nil

View file

@ -3,7 +3,7 @@ package netmap
import ( import (
"errors" "errors"
"fmt" "fmt"
"sort" "slices"
"strconv" "strconv"
"strings" "strings"
@ -227,14 +227,6 @@ func (x NodeInfo) Hash() uint64 {
return hrw.Hash(x.m.GetPublicKey()) return hrw.Hash(x.m.GetPublicKey())
} }
// less declares "less than" comparison between two NodeInfo instances:
// x1 is less than x2 if it has less Hash().
//
// Method is needed for internal placement needs.
func less(x1, x2 NodeInfo) bool {
return x1.Hash() < x2.Hash()
}
func (x *NodeInfo) setNumericAttribute(key string, num uint64) { func (x *NodeInfo) setNumericAttribute(key string, num uint64) {
x.SetAttribute(key, strconv.FormatUint(num, 10)) x.SetAttribute(key, strconv.FormatUint(num, 10))
} }
@ -455,15 +447,11 @@ func (x *NodeInfo) SortAttributes() {
return return
} }
sort.Slice(as, func(i, j int) bool { slices.SortFunc(as, func(ai, aj netmap.Attribute) int {
switch strings.Compare(as[i].GetKey(), as[j].GetKey()) { if r := strings.Compare(ai.GetKey(), aj.GetKey()); r != 0 {
case -1: return r
return true
case 1:
return false
default:
return as[i].GetValue() < as[j].GetValue()
} }
return strings.Compare(ai.GetValue(), aj.GetValue())
}) })
x.m.SetAttributes(as) x.m.SetAttributes(as)

View file

@ -1,8 +1,9 @@
package netmap package netmap
import ( import (
"cmp"
"fmt" "fmt"
"sort" "slices"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
"git.frostfs.info/TrueCloudLab/hrw" "git.frostfs.info/TrueCloudLab/hrw"
@ -70,12 +71,12 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) {
// we also need to have deterministic input to HRW sorting routine. // we also need to have deterministic input to HRW sorting routine.
if len(c.hrwSeed) == 0 { if len(c.hrwSeed) == 0 {
if s.GetAttribute() == "" { if s.GetAttribute() == "" {
sort.Slice(buckets, func(i, j int) bool { slices.SortFunc(buckets, func(b1, b2 nodeAttrPair) int {
return less(buckets[i].nodes[0], buckets[j].nodes[0]) return cmp.Compare(b1.nodes[0].Hash(), b2.nodes[0].Hash())
}) })
} else { } else {
sort.Slice(buckets, func(i, j int) bool { slices.SortFunc(buckets, func(b1, b2 nodeAttrPair) int {
return buckets[i].attr < buckets[j].attr return cmp.Compare(b1.attr, b2.attr)
}) })
} }
} }
@ -103,8 +104,10 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) {
if len(c.hrwSeed) != 0 { if len(c.hrwSeed) != 0 {
weights := make([]float64, len(res)) weights := make([]float64, len(res))
a := new(meanIQRAgg)
for i := range res { for i := range res {
weights[i] = calcBucketWeight(res[i], newMeanIQRAgg(), c.weightFunc) a.clear()
weights[i] = calcBucketWeight(res[i], a, c.weightFunc)
} }
hrw.SortHasherSliceByWeightValue(res, weights, c.hrwSeedHash) hrw.SortHasherSliceByWeightValue(res, weights, c.hrwSeedHash)
@ -168,8 +171,10 @@ func (c *context) getSelectionBase(s netmap.Selector) []nodeAttrPair {
} }
if len(c.hrwSeed) != 0 { if len(c.hrwSeed) != 0 {
var ws []float64
for i := range result { for i := range result {
hrw.SortHasherSliceByWeightValue(result[i].nodes, result[i].nodes.weights(c.weightFunc), c.hrwSeedHash) ws = result[i].nodes.appendWeightsTo(c.weightFunc, ws[:0])
hrw.SortHasherSliceByWeightValue(result[i].nodes, ws, c.hrwSeedHash)
} }
} }

View file

@ -1,11 +1,12 @@
package netmap package netmap
import ( import (
"cmp"
"crypto/rand" "crypto/rand"
"encoding/binary" "encoding/binary"
"fmt" "fmt"
mrand "math/rand" mrand "math/rand"
"sort" "slices"
"strconv" "strconv"
"testing" "testing"
@ -85,8 +86,8 @@ func BenchmarkHRWSort(b *testing.B) {
copy(realNodes, vectors) copy(realNodes, vectors)
b.StartTimer() b.StartTimer()
sort.Slice(vectors, func(i, j int) bool { slices.SortFunc(vectors, func(vi, vj nodes) int {
return less(vectors[i][0], vectors[j][0]) return cmp.Compare(vi[0].Hash(), vj[0].Hash())
}) })
hrw.SortSliceByWeightIndex(realNodes, weights, pivot) hrw.SortSliceByWeightIndex(realNodes, weights, pivot)
} }

View file

@ -1,6 +1,7 @@
package session_test package session_test
import ( import (
"bytes"
"crypto/rand" "crypto/rand"
"fmt" "fmt"
"math" "math"
@ -17,7 +18,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/nspcc-dev/neo-go/pkg/util/slice"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -179,7 +179,7 @@ func TestContainerProtocolV2(t *testing.T) {
breakSign: func(m *v2session.Token) { breakSign: func(m *v2session.Token) {
body := m.GetBody() body := m.GetBody()
key := body.GetSessionKey() key := body.GetSessionKey()
cp := slice.Copy(key) cp := bytes.Clone(key)
cp[len(cp)-1]++ cp[len(cp)-1]++
body.SetSessionKey(cp) body.SetSessionKey(cp)
}, },

View file

@ -1,6 +1,7 @@
package session_test package session_test
import ( import (
"bytes"
"crypto/ecdsa" "crypto/ecdsa"
"fmt" "fmt"
"math" "math"
@ -19,7 +20,6 @@ import (
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/util/slice"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -200,7 +200,7 @@ func TestObjectProtocolV2(t *testing.T) {
breakSign: func(m *v2session.Token) { breakSign: func(m *v2session.Token) {
body := m.GetBody() body := m.GetBody()
key := body.GetSessionKey() key := body.GetSessionKey()
cp := slice.Copy(key) cp := bytes.Clone(key)
cp[len(cp)-1]++ cp[len(cp)-1]++
body.SetSessionKey(cp) body.SetSessionKey(cp)
}, },

View file

@ -1,6 +1,7 @@
package user_test package user_test
import ( import (
"bytes"
"crypto/rand" "crypto/rand"
"testing" "testing"
@ -10,7 +11,6 @@ import (
"github.com/mr-tron/base58" "github.com/mr-tron/base58"
"github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/encoding/address"
"github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/util/slice"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -84,7 +84,7 @@ func TestV2_ID(t *testing.T) {
}) })
t.Run("invalid prefix", func(t *testing.T) { t.Run("invalid prefix", func(t *testing.T) {
val := slice.Copy(val) val := bytes.Clone(val)
val[0]++ val[0]++
m.SetValue(val) m.SetValue(val)
@ -94,7 +94,7 @@ func TestV2_ID(t *testing.T) {
}) })
t.Run("invalid checksum", func(t *testing.T) { t.Run("invalid checksum", func(t *testing.T) {
val := slice.Copy(val) val := bytes.Clone(val)
val[21]++ val[21]++
m.SetValue(val) m.SetValue(val)