ale64bit/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#539] getsvc: Use buffer to assemble object

Browse source 
To reduce memory consumption.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-07-24 13:40:47 +03:00
parent 32c77f3a23
commit 286242cad0
4 changed files with 20 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
pkg/services/object/get/assemble.go
View file

@ -113,8 +113,9 @@ func (r *request) HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object,
return w.Object(), nil return w.Object(), nil
} }
func (r *request) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range) (*objectSDK.Object, error) { func (r *request) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, payloadBuffer []byte) (*objectSDK.Object, error) {
w := NewSimpleObjectWriter() w := NewSimpleObjectWriter()
w.SetPayloadBuffer(payloadBuffer)
p := r.prm p := r.prm
p.common = p.common.WithLocalOnly(false) p.common = p.common.WithLocalOnly(false)

13
pkg/services/object/get/assembler.go
View file

@ -10,7 +10,7 @@ import (
) )
type objectGetter interface { type objectGetter interface {
GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range) (*objectSDK.Object, error) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, buffer []byte) (*objectSDK.Object, error)
HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object, error) HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object, error)
} }
@ -77,7 +77,7 @@ func (a *assembler) getLastPartOrLinkObjectID() (oid.ID, bool) {
} }
func (a *assembler) initializeFromSourceObjectID(ctx context.Context, id oid.ID) (*oid.ID, []oid.ID, error) { func (a *assembler) initializeFromSourceObjectID(ctx context.Context, id oid.ID) (*oid.ID, []oid.ID, error) {
sourceObject, err := a.getChildObject(ctx, id, nil, true) sourceObject, err := a.getChildObject(ctx, id, nil, true, nil)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -131,8 +131,8 @@ func (a *assembler) initializeFromSourceObjectID(ctx context.Context, id oid.ID)
return nil, sourceObject.Children(), nil return nil, sourceObject.Children(), nil
} }
func (a *assembler) getChildObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, verifyIsChild bool) (*objectSDK.Object, error) { func (a *assembler) getChildObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, verifyIsChild bool, payloadBuffer []byte) (*objectSDK.Object, error) {
obj, err := a.objGetter.GetObject(ctx, id, rng) obj, err := a.objGetter.GetObject(ctx, id, rng, payloadBuffer)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -176,13 +176,14 @@ func (a *assembler) assemleObjectByPreviousIDInReverse(ctx context.Context, prev
func (a *assembler) assemblePayloadByObjectIDs(ctx context.Context, writer ObjectWriter, partIDs []oid.ID, partRanges []objectSDK.Range, verifyIsChild bool) error { func (a *assembler) assemblePayloadByObjectIDs(ctx context.Context, writer ObjectWriter, partIDs []oid.ID, partRanges []objectSDK.Range, verifyIsChild bool) error {
withRng := len(partRanges) > 0 && a.rng != nil withRng := len(partRanges) > 0 && a.rng != nil
var buffer []byte
for i := range partIDs { for i := range partIDs {
var r *objectSDK.Range var r *objectSDK.Range
if withRng { if withRng {
r = &partRanges[i] r = &partRanges[i]
} }
child, err := a.getChildObject(ctx, partIDs[i], r, verifyIsChild) child, err := a.getChildObject(ctx, partIDs[i], r, verifyIsChild, buffer)
if err != nil { if err != nil {
return err return err
} }
@ -190,6 +191,8 @@ func (a *assembler) assemblePayloadByObjectIDs(ctx context.Context, writer Objec
if err := writer.WriteChunk(ctx, child.Payload()); err != nil { if err := writer.WriteChunk(ctx, child.Payload()); err != nil {
return err return err
} }
buffer = child.Payload()
} }
return nil return nil
} }

1
pkg/services/object/get/get_test.go
View file

@ -629,6 +629,7 @@ func TestGetRemoteSmall(t *testing.T) {
*c1, *c2 = *c2, *c1 *c1, *c2 = *c2, *c1
w.SetPayloadBuffer(make([]byte, 0))
err = svc.Get(ctx, p) err = svc.Get(ctx, p)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, obj, w.Object()) require.Equal(t, obj, w.Object())

8
pkg/services/object/get/writer.go
View file

@ -49,10 +49,18 @@ func NewSimpleObjectWriter() *SimpleObjectWriter {
} }
} }
func (s *SimpleObjectWriter) SetPayloadBuffer(buffer []byte) {
if buffer != nil {
s.pld = buffer[:0]
}
}
func (s *SimpleObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Object) error { func (s *SimpleObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Object) error {
s.obj = obj s.obj = obj
if s.pld == nil {
s.pld = make([]byte, 0, obj.PayloadSize()) s.pld = make([]byte, 0, obj.PayloadSize())
}
return nil return nil
} }