From 307355f16507d27df26bd76bb235cd2e50b0fe6d Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Wed, 10 Feb 2021 19:52:02 +0300 Subject: [PATCH] [#316] locode/airports: Scan csv table into memory Scanning csv-table entries one-by-one takes significant time and system resources. To speed up random access to table records, on the first call, the table is pumped into memory (map). On subsequent calls, I/O operations are not performed. Signed-off-by: Leonard Lyubich --- pkg/util/locode/db/airports/calls.go | 131 ++++++++++++++------------- pkg/util/locode/db/airports/db.go | 7 ++ 2 files changed, 76 insertions(+), 62 deletions(-) diff --git a/pkg/util/locode/db/airports/calls.go b/pkg/util/locode/db/airports/calls.go index f908f9dc9..33d1fd783 100644 --- a/pkg/util/locode/db/airports/calls.go +++ b/pkg/util/locode/db/airports/calls.go @@ -40,68 +40,43 @@ type record struct { lng string } -// Get scans records of the OpenFlights Airport table one-by-one, and -// returns entry that matches passed UN/LOCODE record. +// Get scans records of the OpenFlights Airport to in-memory table (once), +// and returns entry that matches passed UN/LOCODE record. // // Records are matched if they have the same country code and either // same IATA code or same city name (location name in UN/LOCODE). // // Returns locodedb.ErrAirportNotFound if no entry matches. -func (db *DB) Get(locodeRecord locode.Record) (rec *locodedb.AirportRecord, err error) { - err = db.scanWords(db.airports, airportFldNum, func(words []string) error { - airportRecord := record{ - city: words[airportCity], - country: words[airportCountry], - iata: words[airportIATA], - lat: words[airportLatitude], - lng: words[airportLongitude], +func (db *DB) Get(locodeRecord locode.Record) (*locodedb.AirportRecord, error) { + if err := db.initAirports(); err != nil { + return nil, err + } + + records := db.mAirports[locodeRecord.LOCODE.CountryCode()] + + for i := range records { + if locodeRecord.LOCODE.LocationCode() != records[i].iata && + locodeRecord.NameWoDiacritics != records[i].city { + continue } - if related, err := db.isRelated(locodeRecord, airportRecord); err != nil || !related { - return err - } - - lat, err := strconv.ParseFloat(airportRecord.lat, 64) + lat, err := strconv.ParseFloat(records[i].lat, 64) if err != nil { - return err + return nil, err } - lng, err := strconv.ParseFloat(airportRecord.lng, 64) + lng, err := strconv.ParseFloat(records[i].lng, 64) if err != nil { - return err + return nil, err } - rec = &locodedb.AirportRecord{ - CountryName: airportRecord.country, + return &locodedb.AirportRecord{ + CountryName: records[i].country, Point: locodedb.NewPoint(lat, lng), - } - - return errScanInt - }) - - if err == nil && rec == nil { - err = locodedb.ErrAirportNotFound + }, nil } - return -} - -func (db *DB) isRelated(locodeRecord locode.Record, airportRecord record) (bool, error) { - countryCode, err := db.countryCode(airportRecord.country) - if err != nil { - return false, errors.Wrap(err, "could not read country code of airport") - } - - sameCountry := locodeRecord.LOCODE.CountryCode() == countryCode - sameIATA := locodeRecord.LOCODE.LocationCode() == airportRecord.iata - - if sameCountry && sameIATA { - return true, nil - } - - sameCity := locodeRecord.Name == airportRecord.city - - return sameCountry && sameCity, nil + return nil, locodedb.ErrAirportNotFound } const ( @@ -114,35 +89,67 @@ const ( countryFldNum ) -// CountryName scans records of the OpenFlights Country table, and returns -// name of the country by code. +// CountryName scans records of the OpenFlights Country table to in-memory table (once), +// and returns name of the country by code. // // Returns locodedb.ErrCountryNotFound if no entry matches. func (db *DB) CountryName(code *locodedb.CountryCode) (name string, err error) { - err = db.scanWords(db.countries, countryFldNum, func(words []string) error { - if words[countryISOCode] == code.String() { - name = words[countryName] - return errScanInt + if err = db.initCountries(); err != nil { + return + } + + argCode := code.String() + + for cName, cCode := range db.mCountries { + if cCode == argCode { + name = cName + break } + } - return nil - }) - - if err == nil && name == "" { + if name == "" { err = locodedb.ErrCountryNotFound } return } -func (db *DB) countryCode(country string) (code string, err error) { - err = db.scanWords(db.countries, countryFldNum, func(words []string) error { - if words[countryName] == country { - code = words[countryISOCode] - return errScanInt +func (db *DB) initAirports() (err error) { + db.airportsOnce.Do(func() { + db.mAirports = make(map[string][]record) + + if err = db.initCountries(); err != nil { + return } - return nil + err = db.scanWords(db.airports, airportFldNum, func(words []string) error { + countryCode := db.mCountries[words[airportCountry]] + if countryCode != "" { + db.mAirports[countryCode] = append(db.mAirports[countryCode], record{ + city: words[airportCity], + country: words[airportCountry], + iata: words[airportIATA], + lat: words[airportLatitude], + lng: words[airportLongitude], + }) + } + + return nil + }) + }) + + return +} + +func (db *DB) initCountries() (err error) { + db.countriesOnce.Do(func() { + db.mCountries = make(map[string]string) + + err = db.scanWords(db.countries, countryFldNum, func(words []string) error { + db.mCountries[words[countryName]] = words[countryISOCode] + + return nil + }) }) return diff --git a/pkg/util/locode/db/airports/db.go b/pkg/util/locode/db/airports/db.go index 046d9aac5..d63aaa518 100644 --- a/pkg/util/locode/db/airports/db.go +++ b/pkg/util/locode/db/airports/db.go @@ -3,6 +3,7 @@ package airportsdb import ( "fmt" "os" + "sync" ) // Prm groups the required parameters of the DB's constructor. @@ -30,6 +31,12 @@ type Prm struct { // The DB is immediately ready to work through API. type DB struct { airports, countries pathMode + + airportsOnce, countriesOnce sync.Once + + mCountries map[string]string + + mAirports map[string][]record } type pathMode struct {