realloc/frostfs-sdk-go
1
0
Fork 0
forked from TrueCloudLab/frostfs-sdk-go
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#188] transformer: Allow to provide size hint

Browse source 
For big objects with known size we can optimize allocation patterns
by providing size hint. As with any hint, it does not affect transformer
functionality: slices with capacity > MaxSize are never allocated.

```
goos: linux
goarch: amd64
pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer
cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz
                                                │     out     │
                                                │   sec/op    │
Transformer/small/no_size_hint-8                  65.44µ ± 3%
Transformer/small/no_size_hint,_with_buffer-8     64.24µ ± 5%
Transformer/small/with_size_hint,_with_buffer-8   58.70µ ± 5%
Transformer/big/no_size_hint-8                    367.8m ± 3%
Transformer/big/no_size_hint,_with_buffer-8       562.7m ± 0%
Transformer/big/with_size_hint,_with_buffer-8     385.6m ± 7%
geomean                                           5.197m

                                                │     out      │
                                                │     B/op     │
Transformer/small/no_size_hint-8                  13.40Ki ± 0%
Transformer/small/no_size_hint,_with_buffer-8     13.40Ki ± 0%
Transformer/small/with_size_hint,_with_buffer-8   13.39Ki ± 0%
Transformer/big/no_size_hint-8                    288.0Mi ± 0%
Transformer/big/no_size_hint,_with_buffer-8       1.390Gi ± 0%
Transformer/big/with_size_hint,_with_buffer-8     288.0Mi ± 0%
geomean                                           2.533Mi

                                                │    out     │
                                                │ allocs/op  │
Transformer/small/no_size_hint-8                  92.00 ± 0%
Transformer/small/no_size_hint,_with_buffer-8     92.00 ± 0%
Transformer/small/with_size_hint,_with_buffer-8   92.00 ± 0%
Transformer/big/no_size_hint-8                    546.5 ± 0%
Transformer/big/no_size_hint,_with_buffer-8       607.5 ± 0%
Transformer/big/with_size_hint,_with_buffer-8     545.5 ± 0%
geomean                                           228.1
```

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Evgenii Stratonikov 2023-10-26 18:13:54 +03:00 • committed by Evgenii Stratonikov
parent a02c0bfac8
commit 665e5807bc
4 changed files with 134 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

72
object/transformer/size_hint_test.go Normal file
View file

@ -0,0 +1,72 @@
package transformer
import (
"context"
"crypto/rand"
"math"
"testing"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/stretchr/testify/require"
)
func TestTransformerSizeHintCorrectness(t *testing.T) {
const (
maxSize = 100
payloadSize = maxSize*2 + maxSize/2
)
pk, err := keys.NewPrivateKey()
require.NoError(t, err)
p := Params{
Key: &pk.PrivateKey,
NetworkState: dummyEpochSource(123),
MaxSize: maxSize,
WithoutHomomorphicHash: true,
}
cnr := cidtest.ID()
hdr := newObject(cnr)
var owner user.ID
user.IDFromKey(&owner, pk.PrivateKey.PublicKey)
hdr.SetOwnerID(&owner)
expected := make([]byte, payloadSize)
_, _ = rand.Read(expected)
t.Run("default", func(t *testing.T) {
p.SizeHint = 0
testPayloadEqual(t, p, hdr, expected)
})
t.Run("size hint is perfect", func(t *testing.T) {
p.SizeHint = payloadSize
testPayloadEqual(t, p, hdr, expected)
})
t.Run("size hint < payload size", func(t *testing.T) {
p.SizeHint = payloadSize / 2
testPayloadEqual(t, p, hdr, expected)
})
t.Run("size hint > payload size", func(t *testing.T) {
p.SizeHint = math.MaxUint64
testPayloadEqual(t, p, hdr, expected)
})
}
func testPayloadEqual(t *testing.T, p Params, hdr *objectSDK.Object, expected []byte) {
tt := new(testTarget)
p.NextTargetInit = func() ObjectWriter { return tt }
target := NewPayloadSizeLimiter(p)
writeObject(t, context.Background(), target, hdr, expected)
var actual []byte
for i := range tt.objects {
actual = append(actual, tt.objects[i].Payload()...)
}
require.Equal(t, expected, actual)
}