Realloc slice exponentially in mfs

`registry/storage/driver/inmemory/driver_test.go` times out after ~10min. The slow test is `testsuites.go:TestWriteReadLargeStreams()` which writes a 5GB file.
Root cause is inefficient slice reallocation algorithm. The slice holding file bytes grows only 32K on each allocation. To fix it, this PR grows slice exponentially.

Signed-off-by: Wei Meng <wemeng@microsoft.com>
This commit is contained in:
Wei Meng 2022-07-11 16:38:35 +08:00
parent 8857a19487
commit 35cae1099e

View file

@ -290,18 +290,28 @@ func (f *file) ReadAt(p []byte, offset int64) (n int, err error) {
return copy(p, f.data[offset:]), nil return copy(p, f.data[offset:]), nil
} }
// reallocExponent is the exponent used to realloc a slice. The value roughly
// follows the behavior of Go built-in append function.
const reallocExponent = 1.25
func (f *file) WriteAt(p []byte, offset int64) (n int, err error) { func (f *file) WriteAt(p []byte, offset int64) (n int, err error) {
off := int(offset) newLen := offset + int64(len(p))
if cap(f.data) < off+len(p) { if int64(cap(f.data)) < newLen {
data := make([]byte, len(f.data), off+len(p)) // Grow slice exponentially to ensure amortized linear time complexity
// of reallocation
newCap := int64(float64(cap(f.data)) * reallocExponent)
if newCap < newLen {
newCap = newLen
}
data := make([]byte, len(f.data), newCap)
copy(data, f.data) copy(data, f.data)
f.data = data f.data = data
} }
f.mod = time.Now() f.mod = time.Now()
f.data = f.data[:off+len(p)] f.data = f.data[:newLen]
return copy(f.data[off:off+len(p)], p), nil return copy(f.data[offset:newLen], p), nil
} }
func (f *file) String() string { func (f *file) String() string {