TrueCloudLab/frostfs-sdk-go
30
0
Fork 21
Code Issues 29 Pull requests 2 Releases Activity Actions

Use typed HRW methods #21

Merged
dstepanov-yadro merged 2 commits from feat/hrw-optimize into master 2023-02-28 15:07:15 +00:00
Conversation 0 Commits 2 Files changed 6 +13 -7
6 changed files with 13 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -5,7 +5,7 @@ go 1.18
require ( require (
github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68 github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68
github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42 github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42
github.com/TrueCloudLab/hrw v1.1.0 github.com/TrueCloudLab/hrw v1.1.1-0.20230227111858-79b208bebf52
github.com/TrueCloudLab/tzhash v1.7.0 github.com/TrueCloudLab/tzhash v1.7.0
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12
github.com/google/uuid v1.3.0 github.com/google/uuid v1.3.0

BIN
go.sum
View file

Binary file not shown.

2
netmap/netmap.go
View file

@ -152,7 +152,7 @@ func (m NetMap) PlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]NodeIn
for i := range vectors { for i := range vectors {
result[i] = make([]NodeInfo, len(vectors[i])) result[i] = make([]NodeInfo, len(vectors[i]))
copy(result[i], vectors[i]) copy(result[i], vectors[i])
hrw.SortSliceByWeightValue(result[i], nodes(result[i]).weights(wf), h) hrw.SortHasherSliceByWeightValue(result[i], nodes(result[i]).weights(wf), h)
} }
fyrchik commented 2023-02-27 14:54:23 +00:00 (Migrated from github.com)
Review
Copy link

Can we do an optimization with hash precalculation, similar to what you've done in the frostfs-node?

Should be done in readFromV2 and SetPublicKey functions. We should also fallback to a normal calculation if the hash is 0. Am I missing something here? @carpawell

Can we do an optimization with hash precalculation, similar to what you've done in the `frostfs-node`? Should be done in `readFromV2` and `SetPublicKey` functions. We should also fallback to a normal calculation if the hash is 0. Am I missing something here? @carpawell
dstepanov-yadro commented 2023-02-27 15:41:19 +00:00 (Migrated from github.com)
Review
Copy link

done

done
fyrchik commented 2023-02-28 07:37:34 +00:00 (Migrated from github.com)
Review
Copy link

Could you mention about this in the commit message? Actually, a separate commit for this specific opitimization would be nice.

Could you mention about this in the commit message? Actually, a separate commit for this specific opitimization would be nice.
dstepanov-yadro commented 2023-02-28 11:07:07 +00:00 (Migrated from github.com)
Review
Copy link

done

done
return result, nil return result, nil

8
netmap/node_info.go
View file

@ -25,7 +25,8 @@ import (
// //
// Instances can be created using built-in var declaration. // Instances can be created using built-in var declaration.
type NodeInfo struct { type NodeInfo struct {
m netmap.NodeInfo m netmap.NodeInfo
hash uint64
} }
// reads NodeInfo from netmap.NodeInfo message. If checkFieldPresence is set, // reads NodeInfo from netmap.NodeInfo message. If checkFieldPresence is set,
@ -86,6 +87,7 @@ func (x *NodeInfo) readFromV2(m netmap.NodeInfo, checkFieldPresence bool) error
} }
x.m = m x.m = m
x.hash = hrw.Hash(binPublicKey)
return nil return nil
} }
@ -167,6 +169,7 @@ func (x *NodeInfo) UnmarshalJSON(data []byte) error {
// See also PublicKey. // See also PublicKey.
func (x *NodeInfo) SetPublicKey(key []byte) { func (x *NodeInfo) SetPublicKey(key []byte) {
x.m.SetPublicKey(key) x.m.SetPublicKey(key)
x.hash = hrw.Hash(x.m.GetPublicKey())
} }
// PublicKey returns value set using SetPublicKey. // PublicKey returns value set using SetPublicKey.
@ -233,6 +236,9 @@ var _ hrw.Hasher = NodeInfo{}
// Hash is needed to support weighted HRW therefore sort function sorts nodes // Hash is needed to support weighted HRW therefore sort function sorts nodes
// based on their public key. Hash isn't expected to be used directly. // based on their public key. Hash isn't expected to be used directly.
func (x NodeInfo) Hash() uint64 { func (x NodeInfo) Hash() uint64 {
if x.hash != 0 {
return x.hash
}
return hrw.Hash(x.m.GetPublicKey()) return hrw.Hash(x.m.GetPublicKey())
} }

4
netmap/selector.go
View file

@ -108,7 +108,7 @@ func (c *context) getSelection(p PlacementPolicy, s netmap.Selector) ([]nodes, e
weights[i] = calcBucketWeight(res[i], newMeanIQRAgg(), c.weightFunc) weights[i] = calcBucketWeight(res[i], newMeanIQRAgg(), c.weightFunc)
} }
hrw.SortSliceByWeightValue(res, weights, c.hrwSeedHash) hrw.SortHasherSliceByWeightValue(res, weights, c.hrwSeedHash)
} }
if s.GetAttribute() == "" { if s.GetAttribute() == "" {
@ -164,7 +164,7 @@ func (c *context) getSelectionBase(subnetID subnetid.ID, s netmap.Selector) []no
if len(c.hrwSeed) != 0 { if len(c.hrwSeed) != 0 {
for i := range result { for i := range result {
hrw.SortSliceByWeightValue(result[i].nodes, result[i].nodes.weights(c.weightFunc), c.hrwSeedHash) hrw.SortHasherSliceByWeightValue(result[i].nodes, result[i].nodes.weights(c.weightFunc), c.hrwSeedHash)
} }
} }

4
netmap/selector_test.go
View file

@ -50,7 +50,7 @@ func BenchmarkHRWSort(b *testing.B) {
copy(realNodes, vectors) copy(realNodes, vectors)
b.StartTimer() b.StartTimer()
hrw.SortSliceByValue(realNodes, pivot) hrw.SortHasherSliceByValue(realNodes, pivot)
} }
}) })
b.Run("only sort by index", func(b *testing.B) { b.Run("only sort by index", func(b *testing.B) {
@ -72,7 +72,7 @@ func BenchmarkHRWSort(b *testing.B) {
copy(realNodes, vectors) copy(realNodes, vectors)
b.StartTimer() b.StartTimer()
hrw.SortSliceByWeightValue(realNodes, weights, pivot) hrw.SortHasherSliceByWeightValue(realNodes, weights, pivot)
} }
}) })
b.Run("sort by ID, then by index (deterministic)", func(b *testing.B) { b.Run("sort by ID, then by index (deterministic)", func(b *testing.B) {