package hrw import ( "encoding/binary" "fmt" "hash/fnv" "reflect" "strconv" "testing" "github.com/reusee/mmh3" ) type hashString string var testKey = []byte("Golang simple HRW implementation") func Example() { // given a set of servers servers := []string{ "one.example.com", "two.example.com", "three.example.com", "four.example.com", "five.example.com", "six.example.com", } // HRW can consistently select a uniformly-distributed set of servers for // any given key var ( key = []byte("/examples/object-key") h = hash(key) err = SortSliceByValue(servers, h) ) if err != nil { panic(err) } for id := range servers { fmt.Printf("trying GET %s%s\n", servers[id], key) } // Output: // trying GET six.example.com/examples/object-key // trying GET one.example.com/examples/object-key // trying GET three.example.com/examples/object-key // trying GET four.example.com/examples/object-key // trying GET five.example.com/examples/object-key // trying GET two.example.com/examples/object-key } func (h hashString) Hash() uint64 { hs := fnv.New64() // error always nil _, _ = hs.Write([]byte(h)) return (hs.Sum64() >> 1) % m64 } func hash(key []byte) uint64 { h := fnv.New64() // error always nil _, _ = h.Write(key) return (h.Sum64() >> 1) ^ m64 } func mur3hash(key []byte) uint64 { h := mmh3.New128() // error always nil _, _ = h.Write(key) var ( data = h.Sum(nil) length = len(data) result uint64 ) for i := 0; i < length; i++ { result += uint64(data[i]) << uint64(length-i) } return result } func TestSortSliceByIndex(t *testing.T) { actual := []string{"a", "b", "c", "d", "e", "f"} expect := []string{"e", "a", "c", "d", "b", "f"} hash := hash(testKey) SortSliceByIndex(actual, hash) if !reflect.DeepEqual(actual, expect) { t.Errorf("Was %#v, but expected %#v", actual, expect) } } func TestSortSliceByValue(t *testing.T) { actual := []string{"a", "b", "c", "d", "e", "f"} expect := []string{"e", "b", "c", "d", "f", "a"} hash := hash(testKey) if err := SortSliceByValue(actual, hash); err != nil { t.Fatal(err) } if !reflect.DeepEqual(actual, expect) { t.Errorf("Was %#v, but expected %#v", actual, expect) } } func TestSortSliceByValueFail(t *testing.T) { t.Run("empty slice", func(t *testing.T) { actual := make([]int, 0) hash := hash(testKey) if err := SortSliceByValue(actual, hash); err != nil { t.Fatal(err) } }) t.Run("must be slice", func(t *testing.T) { actual := 10 hash := hash(testKey) if err := SortSliceByValue(actual, hash); err == nil { t.Fatal("must fail for bad type") } }) t.Run("must fail for unknown type", func(t *testing.T) { actual := []byte{1, 2, 3, 4, 5} hash := hash(testKey) if err := SortSliceByValue(actual, hash); err == nil { t.Fatal("must fail for bad type") } }) } func TestSortSliceByValueHasher(t *testing.T) { actual := []hashString{"a", "b", "c", "d", "e", "f"} expect := []hashString{"e", "d", "c", "a", "b", "f"} hash := hash(testKey) if err := SortSliceByValue(actual, hash); err != nil { t.Fatal(err) } if !reflect.DeepEqual(actual, expect) { t.Errorf("Was %#v, but expected %#v", actual, expect) } } func TestSortSliceByValueIntSlice(t *testing.T) { actual := []int{0, 1, 2, 3, 4, 5} expect := []int{2, 0, 5, 3, 4, 1} hash := hash(testKey) if err := SortSliceByValue(actual, hash); err != nil { t.Fatal(err) } if !reflect.DeepEqual(actual, expect) { t.Errorf("Was %#v, but expected %#v", actual, expect) } } func TestSortByWeight(t *testing.T) { nodes := []uint64{1, 2, 3, 4, 5} hash := mur3hash(testKey) actual := SortByWeight(nodes, hash) expected := []uint64{0, 1, 4, 2, 3} if !reflect.DeepEqual(actual, expected) { t.Errorf("Was %#v, but expected %#v", actual, expected) } } func TestUniformDistribution(t *testing.T) { var ( i uint64 size = uint64(4) nodes = make([]uint64, 0, size) counts = make(map[uint64]uint64) key = make([]byte, 16) keys = uint64(10000000) ) for i = 0; i < size; i++ { nodes = append(nodes, i) } for i = 0; i < keys; i++ { binary.BigEndian.PutUint64(key, i) hash := hash(key) counts[SortByWeight(nodes, hash)[0]]++ } mean := float64(keys) / float64(len(nodes)) delta := mean * 0.002 // 0.2% for node, count := range counts { d := mean - float64(count) if d > delta || (0-d) > delta { t.Errorf( "Node %d received %d keys, expected %v (+/- %v)", node, count, mean, delta, ) } } } func BenchmarkSortByWeight_fnv_10(b *testing.B) { hash := hash(testKey) _ = benchmarkSortByWeight(b, 10, hash) } func BenchmarkSortByWeight_fnv_100(b *testing.B) { hash := hash(testKey) _ = benchmarkSortByWeight(b, 100, hash) } func BenchmarkSortByWeight_fnv_1000(b *testing.B) { hash := hash(testKey) _ = benchmarkSortByWeight(b, 1000, hash) } func BenchmarkSortByIndex_fnv_10(b *testing.B) { hash := hash(testKey) benchmarkSortByIndex(b, 10, hash) } func BenchmarkSortByIndex_fnv_100(b *testing.B) { hash := hash(testKey) benchmarkSortByIndex(b, 100, hash) } func BenchmarkSortByIndex_fnv_1000(b *testing.B) { hash := hash(testKey) benchmarkSortByIndex(b, 1000, hash) } func BenchmarkSortByValue_fnv_10(b *testing.B) { hash := hash(testKey) benchmarkSortByValue(b, 10, hash) } func BenchmarkSortByValue_fnv_100(b *testing.B) { hash := hash(testKey) benchmarkSortByValue(b, 100, hash) } func BenchmarkSortByValue_fnv_1000(b *testing.B) { hash := hash(testKey) benchmarkSortByValue(b, 1000, hash) } func benchmarkSortByWeight(b *testing.B, n int, hash uint64) uint64 { servers := make([]uint64, n) for i := uint64(0); i < uint64(len(servers)); i++ { servers[i] = i } b.ResetTimer() b.ReportAllocs() var x uint64 for i := 0; i < b.N; i++ { x += SortByWeight(servers, hash)[0] } return x } func benchmarkSortByIndex(b *testing.B, n int, hash uint64) { servers := make([]uint64, n) for i := uint64(0); i < uint64(len(servers)); i++ { servers[i] = i } b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { SortSliceByIndex(servers, hash) } } func benchmarkSortByValue(b *testing.B, n int, hash uint64) { servers := make([]string, n) for i := uint64(0); i < uint64(len(servers)); i++ { servers[i] = "localhost:" + strconv.FormatUint(60000-i, 10) } b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { if err := SortSliceByValue(servers, hash); err != nil { b.Fatal(err) } } }