realloc/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Releases Packages Activity

[#1462] fstree: Allow to fetch file content lazily

Browse source 
If we should process address based on some condition, there is no need
to read file content in memory.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-05-31 17:11:48 +03:00 • committed by LeL
parent 54d4503701
commit f2a7503964
3 changed files with 37 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

29
pkg/local_object_storage/blobstor/fstree/fstree.go
View file

@ -72,6 +72,7 @@ func addressFromString(s string) (*oid.Address, error) {
type IterationPrm struct {
handler func(addr oid.Address, data []byte) error
ignoreErrors bool
lazyHandler func(oid.Address, func() ([]byte, error)) error
}
// WithHandler sets a function to call on each object.
@ -79,6 +80,13 @@ func (p *IterationPrm) WithHandler(f func(addr oid.Address, data []byte) error)
p.handler = f
}
// WithLazyHandler sets a function to call on each object.
// Second callback parameter opens file and reads all data to a buffer.
// File is not opened at all unless this callback is invoked.
func (p *IterationPrm) WithLazyHandler(f func(oid.Address, func() ([]byte, error)) error) {
p.lazyHandler = f
}
// WithIgnoreErrors sets a flag indicating whether errors should be ignored.
func (p *IterationPrm) WithIgnoreErrors(ignore bool) {
p.ignoreErrors = ignore
@ -124,16 +132,23 @@ func (t *FSTree) iterate(depth int, curPath []string, prm IterationPrm) error {
continue
}
data, err := os.ReadFile(filepath.Join(curPath...))
if err != nil {
if prm.ignoreErrors {
continue
if prm.lazyHandler != nil {
err = prm.lazyHandler(*addr, func() ([]byte, error) {
return os.ReadFile(filepath.Join(curPath...))
})
} else {
var data []byte
data, err = os.ReadFile(filepath.Join(curPath...))
if err != nil {
if prm.ignoreErrors {
continue
}
return err
}
return err
err = prm.handler(*addr, data)
}
if err := prm.handler(*addr, data); err != nil {
// Error occurred in handler, outside of our scope, needs to be reported.
if err != nil {
return err
}
}