forked from TrueCloudLab/frostfs-node
[#309] locode: Parallelize DB generation
For v0.4.0 release: Before: ``` Executed in 571.64 secs fish external usr time 283.07 secs 744.00 micros 283.07 secs sys time 8.41 secs 179.00 micros 8.41 secs ``` After: ``` Executed in 54.23 secs fish external usr time 418.65 secs 1.01 millis 418.65 secs sys time 0.61 secs 0.25 millis 0.60 secs ``` Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
parent
147ae8728a
commit
02c02974b3
3 changed files with 74 additions and 59 deletions
|
@ -9,6 +9,7 @@ Changelog for FrostFS Node
|
||||||
- Reload pprof and metrics on SIGHUP for ir (#125)
|
- Reload pprof and metrics on SIGHUP for ir (#125)
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
- `frostfs-cli util locode generate` is now much faster (#309)
|
||||||
### Fixed
|
### Fixed
|
||||||
- Take network settings into account during netmap contract update (#100)
|
- Take network settings into account during netmap contract update (#100)
|
||||||
- Read config files from dir even if config file not provided via `--config` for node (#238)
|
- Read config files from dir even if config file not provided via `--config` for node (#238)
|
||||||
|
|
|
@ -103,7 +103,7 @@ func recordFromValue(data []byte) (*locodedb.Record, error) {
|
||||||
// Must not be called before successful Open call.
|
// Must not be called before successful Open call.
|
||||||
// Must not be called in read-only mode: behavior is undefined.
|
// Must not be called in read-only mode: behavior is undefined.
|
||||||
func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error {
|
func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error {
|
||||||
return db.bolt.Update(func(tx *bbolt.Tx) error {
|
return db.bolt.Batch(func(tx *bbolt.Tx) error {
|
||||||
countryKey, err := countryBucketKey(key.CountryCode())
|
countryKey, err := countryBucketKey(key.CountryCode())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -3,8 +3,10 @@ package locodedb
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"runtime"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SourceTable is an interface of the UN/LOCODE table.
|
// SourceTable is an interface of the UN/LOCODE table.
|
||||||
|
@ -75,7 +77,20 @@ type NamesDB interface {
|
||||||
|
|
||||||
// FillDatabase generates the FrostFS location database based on the UN/LOCODE table.
|
// FillDatabase generates the FrostFS location database based on the UN/LOCODE table.
|
||||||
func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
|
func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
|
||||||
return table.IterateAll(func(tableRecord locode.Record) error {
|
var errG errgroup.Group
|
||||||
|
|
||||||
|
// Pick some sane default, after this the performance stopped increasing.
|
||||||
|
errG.SetLimit(runtime.NumCPU() * 4)
|
||||||
|
_ = table.IterateAll(func(tableRecord locode.Record) error {
|
||||||
|
errG.Go(func() error {
|
||||||
|
return processTableRecord(tableRecord, airports, continents, names, db)
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return errG.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
|
||||||
if tableRecord.LOCODE.LocationCode() == "" {
|
if tableRecord.LOCODE.LocationCode() == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -149,7 +164,6 @@ func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB
|
||||||
dbRecord.SetContinent(continent)
|
dbRecord.SetContinent(continent)
|
||||||
|
|
||||||
return db.Put(*dbKey, *dbRecord)
|
return db.Put(*dbKey, *dbRecord)
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// LocodeRecord returns the record from the FrostFS location database
|
// LocodeRecord returns the record from the FrostFS location database
|
||||||
|
|
Loading…
Reference in a new issue