CHANGELOG.md

@@ -9,6 +9,7 @@ Changelog for FrostFS Node
 - Reload pprof and metrics on SIGHUP for ir (#125)
 
 ### Changed
+- `frostfs-cli util locode generate` is now much faster (#309)
 ### Fixed
 - Take network settings into account during netmap contract update (#100)
 - Read config files from dir even if config file not provided via `--config` for node (#238)

pkg/util/locode/db/boltdb/calls.go

@@ -103,7 +103,7 @@ func recordFromValue(data []byte) (*locodedb.Record, error) {
 // Must not be called before successful Open call.
 // Must not be called in read-only mode: behavior is undefined.
 func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error {
-	return db.bolt.Update(func(tx *bbolt.Tx) error {
+	return db.bolt.Batch(func(tx *bbolt.Tx) error {
 		countryKey, err := countryBucketKey(key.CountryCode())
 		if err != nil {
 			return err

pkg/util/locode/db/db.go

@@ -3,8 +3,10 @@ package locodedb
 import (
 	"errors"
 	"fmt"
+	"runtime"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
+	"golang.org/x/sync/errgroup"
 )
 
 // SourceTable is an interface of the UN/LOCODE table.
@@ -75,81 +77,93 @@ type NamesDB interface {
 
 // FillDatabase generates the FrostFS location database based on the UN/LOCODE table.
 func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
-	return table.IterateAll(func(tableRecord locode.Record) error {
-		if tableRecord.LOCODE.LocationCode() == "" {
+	var errG errgroup.Group
+
+	// Pick some sane default, after this the performance stopped increasing.
+	errG.SetLimit(runtime.NumCPU() * 4)
+	_ = table.IterateAll(func(tableRecord locode.Record) error {
+		errG.Go(func() error {
+			return processTableRecord(tableRecord, airports, continents, names, db)
+		})
+		return nil
+	})
+	return errG.Wait()
+}
+
+func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
+	if tableRecord.LOCODE.LocationCode() == "" {
+		return nil
+	}
+
+	dbKey, err := NewKey(tableRecord.LOCODE)
+	if err != nil {
+		return err
+	}
+
+	dbRecord, err := NewRecord(tableRecord)
+	if err != nil {
+		if errors.Is(err, errParseCoordinates) {
 			return nil
 		}
 
-		dbKey, err := NewKey(tableRecord.LOCODE)
-		if err != nil {
-			return err
-		}
+		return err
+	}
 
-		dbRecord, err := NewRecord(tableRecord)
+	geoPoint := dbRecord.GeoPoint()
+	countryName := ""
+
+	if geoPoint == nil {
+		airportRecord, err := airports.Get(tableRecord)
 		if err != nil {
-			if errors.Is(err, errParseCoordinates) {
+			if errors.Is(err, ErrAirportNotFound) {
 				return nil
 			}
 
 			return err
 		}
 
-		geoPoint := dbRecord.GeoPoint()
-		countryName := ""
+		geoPoint = airportRecord.Point
+		countryName = airportRecord.CountryName
+	}
 
-		if geoPoint == nil {
-			airportRecord, err := airports.Get(tableRecord)
-			if err != nil {
-				if errors.Is(err, ErrAirportNotFound) {
-					return nil
-				}
+	dbRecord.SetGeoPoint(geoPoint)
 
-				return err
-			}
-
-			geoPoint = airportRecord.Point
-			countryName = airportRecord.CountryName
-		}
-
-		dbRecord.SetGeoPoint(geoPoint)
-
-		if countryName == "" {
-			countryName, err = names.CountryName(dbKey.CountryCode())
-			if err != nil {
-				if errors.Is(err, ErrCountryNotFound) {
-					return nil
-				}
-
-				return err
-			}
-		}
-
-		dbRecord.SetCountryName(countryName)
-
-		if subDivCode := dbRecord.SubDivCode(); subDivCode != "" {
-			subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode)
-			if err != nil {
-				if errors.Is(err, ErrSubDivNotFound) {
-					return nil
-				}
-
-				return err
-			}
-
-			dbRecord.SetSubDivName(subDivName)
-		}
-
-		continent, err := continents.PointContinent(geoPoint)
+	if countryName == "" {
+		countryName, err = names.CountryName(dbKey.CountryCode())
 		if err != nil {
-			return fmt.Errorf("could not calculate continent geo point: %w", err)
-		} else if continent.Is(ContinentUnknown) {
-			return nil
+			if errors.Is(err, ErrCountryNotFound) {
+				return nil
+			}
+
+			return err
+		}
+	}
+
+	dbRecord.SetCountryName(countryName)
+
+	if subDivCode := dbRecord.SubDivCode(); subDivCode != "" {
+		subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode)
+		if err != nil {
+			if errors.Is(err, ErrSubDivNotFound) {
+				return nil
+			}
+
+			return err
 		}
 
-		dbRecord.SetContinent(continent)
+		dbRecord.SetSubDivName(subDivName)
+	}
 
-		return db.Put(*dbKey, *dbRecord)
-	})
+	continent, err := continents.PointContinent(geoPoint)
+	if err != nil {
+		return fmt.Errorf("could not calculate continent geo point: %w", err)
+	} else if continent.Is(ContinentUnknown) {
+		return nil
+	}
+
+	dbRecord.SetContinent(continent)
+
+	return db.Put(*dbKey, *dbRecord)
 }
 
 // LocodeRecord returns the record from the FrostFS location database