CHANGELOG.md

@@ -8,6 +8,7 @@ Changelog for FrostFS Node
 - Change log level on SIGHUP for ir (#125)
 
 ### Changed
+- `frostfs-cli util locode generate` is now much faster (#309)
 ### Fixed
 - Take network settings into account during netmap contract update (#100)
 - Read config files from dir even if config file not provided via `--config` for node (#238)
@@ -17,6 +18,7 @@ Changelog for FrostFS Node
 
 ### Removed
 ### Updated
+- `paulmach/orb` to v0.9.1
 ### Updating from v0.36.0
 
 ## [v0.36.0] - 2023-04-12 - Furtwängler

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/nspcc-dev/neo-go v0.100.1
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/panjf2000/ants/v2 v2.4.0
-	github.com/paulmach/orb v0.2.2
+	github.com/paulmach/orb v0.9.1
 	github.com/prometheus/client_golang v1.15.0
 	github.com/prometheus/client_model v0.3.0
 	github.com/spf13/cast v1.5.0
@@ -94,6 +94,7 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect
 	github.com/twmb/murmur3 v1.1.5 // indirect
 	github.com/urfave/cli v1.22.5 // indirect
+	go.mongodb.org/mongo-driver v1.11.4 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect

go.sum

@@ -290,6 +290,7 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
 github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -337,6 +338,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
@@ -410,9 +412,9 @@ github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/panjf2000/ants/v2 v2.4.0 h1:embKPQeNWMRbnrRKURv4TXJwjQRWMEAfqZT6Pe5hZNc=
 github.com/panjf2000/ants/v2 v2.4.0/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A=
-github.com/paulmach/orb v0.2.2 h1:PblToKAbU0xHVypex/GdZfibA1CeCfN5s0UjxyWExdo=
+github.com/paulmach/orb v0.9.1 h1:NAL1AHClSzudQdZZA1kpjpNhcCEP1SXX+rQviOw0e/g=
-github.com/paulmach/orb v0.2.2/go.mod h1:FkcWtplUAIVqAuhAOV2d3rpbnQyliDOjOcLW9dUrfdU=
+github.com/paulmach/orb v0.9.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
-github.com/paulmach/protoscan v0.2.1-0.20210522164731-4e53c6875432/go.mod h1:2sV+uZ/oQh66m4XJVZm5iqUZ62BN88Ex1E+TTS0nLzI=
+github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
 github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us=
 github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
 github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@@ -489,6 +491,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
@@ -500,6 +503,8 @@ github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNG
 github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
 github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs=
 github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/twmb/murmur3 v1.1.5 h1:i9OLS9fkuLzBXjt6dptlAEyk58fJsSTXbRg3SgVyqgk=
 github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
@@ -507,6 +512,10 @@ github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
 github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo=
 github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -517,6 +526,8 @@ github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBU
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -568,6 +579,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
 golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=

pkg/util/locode/db/boltdb/calls.go

@@ -103,7 +103,7 @@ func recordFromValue(data []byte) (*locodedb.Record, error) {
 // Must not be called before successful Open call.
 // Must not be called in read-only mode: behavior is undefined.
 func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error {
-	return db.bolt.Update(func(tx *bbolt.Tx) error {
+	return db.bolt.Batch(func(tx *bbolt.Tx) error {
 		countryKey, err := countryBucketKey(key.CountryCode())
 		if err != nil {
 			return err

pkg/util/locode/db/db.go

@@ -3,8 +3,10 @@ package locodedb
 import (
 	"errors"
 	"fmt"
+	"runtime"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
+	"golang.org/x/sync/errgroup"
 )
 
 // SourceTable is an interface of the UN/LOCODE table.
@@ -75,81 +77,93 @@ type NamesDB interface {
 
 // FillDatabase generates the FrostFS location database based on the UN/LOCODE table.
 func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
-	return table.IterateAll(func(tableRecord locode.Record) error {
+	var errG errgroup.Group
-		if tableRecord.LOCODE.LocationCode() == "" {
+
+	// Pick some sane default, after this the performance stopped increasing.
+	errG.SetLimit(runtime.NumCPU() * 4)
+	_ = table.IterateAll(func(tableRecord locode.Record) error {
+		errG.Go(func() error {
+			return processTableRecord(tableRecord, airports, continents, names, db)
+		})
+		return nil
+	})
+	return errG.Wait()
+}
+
+func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
+	if tableRecord.LOCODE.LocationCode() == "" {
+		return nil
+	}
+
+	dbKey, err := NewKey(tableRecord.LOCODE)
+	if err != nil {
+		return err
+	}
+
+	dbRecord, err := NewRecord(tableRecord)
+	if err != nil {
+		if errors.Is(err, errParseCoordinates) {
 			return nil
 		}
 
-		dbKey, err := NewKey(tableRecord.LOCODE)
+		return err
-		if err != nil {
+	}
-			return err
-		}
 
-		dbRecord, err := NewRecord(tableRecord)
+	geoPoint := dbRecord.GeoPoint()
+	countryName := ""
+
+	if geoPoint == nil {
+		airportRecord, err := airports.Get(tableRecord)
 		if err != nil {
-			if errors.Is(err, errParseCoordinates) {
+			if errors.Is(err, ErrAirportNotFound) {
 				return nil
 			}
 
 			return err
 		}
 
-		geoPoint := dbRecord.GeoPoint()
+		geoPoint = airportRecord.Point
-		countryName := ""
+		countryName = airportRecord.CountryName
+	}
 
-		if geoPoint == nil {
+	dbRecord.SetGeoPoint(geoPoint)
-			airportRecord, err := airports.Get(tableRecord)
-			if err != nil {
-				if errors.Is(err, ErrAirportNotFound) {
-					return nil
-				}
 
-				return err
+	if countryName == "" {
-			}
+		countryName, err = names.CountryName(dbKey.CountryCode())
-
-			geoPoint = airportRecord.Point
-			countryName = airportRecord.CountryName
-		}
-
-		dbRecord.SetGeoPoint(geoPoint)
-
-		if countryName == "" {
-			countryName, err = names.CountryName(dbKey.CountryCode())
-			if err != nil {
-				if errors.Is(err, ErrCountryNotFound) {
-					return nil
-				}
-
-				return err
-			}
-		}
-
-		dbRecord.SetCountryName(countryName)
-
-		if subDivCode := dbRecord.SubDivCode(); subDivCode != "" {
-			subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode)
-			if err != nil {
-				if errors.Is(err, ErrSubDivNotFound) {
-					return nil
-				}
-
-				return err
-			}
-
-			dbRecord.SetSubDivName(subDivName)
-		}
-
-		continent, err := continents.PointContinent(geoPoint)
 		if err != nil {
-			return fmt.Errorf("could not calculate continent geo point: %w", err)
+			if errors.Is(err, ErrCountryNotFound) {
-		} else if continent.Is(ContinentUnknown) {
+				return nil
-			return nil
+			}
+
+			return err
+		}
+	}
+
+	dbRecord.SetCountryName(countryName)
+
+	if subDivCode := dbRecord.SubDivCode(); subDivCode != "" {
+		subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode)
+		if err != nil {
+			if errors.Is(err, ErrSubDivNotFound) {
+				return nil
+			}
+
+			return err
 		}
 
-		dbRecord.SetContinent(continent)
+		dbRecord.SetSubDivName(subDivName)
+	}
 
-		return db.Put(*dbKey, *dbRecord)
+	continent, err := continents.PointContinent(geoPoint)
-	})
+	if err != nil {
+		return fmt.Errorf("could not calculate continent geo point: %w", err)
+	} else if continent.Is(ContinentUnknown) {
+		return nil
+	}
+
+	dbRecord.SetContinent(continent)
+
+	return db.Put(*dbKey, *dbRecord)
 }
 
 // LocodeRecord returns the record from the FrostFS location database