[#309] locode: Parallelize DB generation
All checks were successful
ci/woodpecker/push/pre-commit Pipeline was successful

For v0.4.0 release:
Before:
```
Executed in  571.64 secs    fish           external
   usr time  283.07 secs  744.00 micros  283.07 secs
   sys time    8.41 secs  179.00 micros    8.41 secs
```

After:
```
Executed in   54.23 secs    fish           external
   usr time  418.65 secs    1.01 millis  418.65 secs
   sys time    0.61 secs    0.25 millis    0.60 secs
```
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Evgenii Stratonikov 2023-05-03 16:23:58 +03:00 committed by Evgenii Stratonikov
parent 147ae8728a
commit 02c02974b3
3 changed files with 74 additions and 59 deletions

View file

@ -9,6 +9,7 @@ Changelog for FrostFS Node
- Reload pprof and metrics on SIGHUP for ir (#125) - Reload pprof and metrics on SIGHUP for ir (#125)
### Changed ### Changed
- `frostfs-cli util locode generate` is now much faster (#309)
### Fixed ### Fixed
- Take network settings into account during netmap contract update (#100) - Take network settings into account during netmap contract update (#100)
- Read config files from dir even if config file not provided via `--config` for node (#238) - Read config files from dir even if config file not provided via `--config` for node (#238)

View file

@ -103,7 +103,7 @@ func recordFromValue(data []byte) (*locodedb.Record, error) {
// Must not be called before successful Open call. // Must not be called before successful Open call.
// Must not be called in read-only mode: behavior is undefined. // Must not be called in read-only mode: behavior is undefined.
func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error { func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error {
return db.bolt.Update(func(tx *bbolt.Tx) error { return db.bolt.Batch(func(tx *bbolt.Tx) error {
countryKey, err := countryBucketKey(key.CountryCode()) countryKey, err := countryBucketKey(key.CountryCode())
if err != nil { if err != nil {
return err return err

View file

@ -3,8 +3,10 @@ package locodedb
import ( import (
"errors" "errors"
"fmt" "fmt"
"runtime"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
"golang.org/x/sync/errgroup"
) )
// SourceTable is an interface of the UN/LOCODE table. // SourceTable is an interface of the UN/LOCODE table.
@ -75,7 +77,20 @@ type NamesDB interface {
// FillDatabase generates the FrostFS location database based on the UN/LOCODE table. // FillDatabase generates the FrostFS location database based on the UN/LOCODE table.
func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
return table.IterateAll(func(tableRecord locode.Record) error { var errG errgroup.Group
// Pick some sane default, after this the performance stopped increasing.
errG.SetLimit(runtime.NumCPU() * 4)
_ = table.IterateAll(func(tableRecord locode.Record) error {
errG.Go(func() error {
return processTableRecord(tableRecord, airports, continents, names, db)
})
return nil
})
return errG.Wait()
}
func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
if tableRecord.LOCODE.LocationCode() == "" { if tableRecord.LOCODE.LocationCode() == "" {
return nil return nil
} }
@ -149,7 +164,6 @@ func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB
dbRecord.SetContinent(continent) dbRecord.SetContinent(continent)
return db.Put(*dbKey, *dbRecord) return db.Put(*dbKey, *dbRecord)
})
} }
// LocodeRecord returns the record from the FrostFS location database // LocodeRecord returns the record from the FrostFS location database