[#316] locode/airports: Scan csv table into memory

Scanning csv-table entries one-by-one takes significant time and system
resources. To speed up random access to table records, on the first call,
the table is pumped into memory (map). On subsequent calls, I/O operations
are not performed.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-02-10 19:52:02 +03:00 committed by Alex Vanin
parent 687c7d3b4a
commit 307355f165
2 changed files with 76 additions and 62 deletions

View file

@ -40,68 +40,43 @@ type record struct {
lng string
}
// Get scans records of the OpenFlights Airport table one-by-one, and
// returns entry that matches passed UN/LOCODE record.
// Get scans records of the OpenFlights Airport to in-memory table (once),
// and returns entry that matches passed UN/LOCODE record.
//
// Records are matched if they have the same country code and either
// same IATA code or same city name (location name in UN/LOCODE).
//
// Returns locodedb.ErrAirportNotFound if no entry matches.
func (db *DB) Get(locodeRecord locode.Record) (rec *locodedb.AirportRecord, err error) {
err = db.scanWords(db.airports, airportFldNum, func(words []string) error {
airportRecord := record{
city: words[airportCity],
country: words[airportCountry],
iata: words[airportIATA],
lat: words[airportLatitude],
lng: words[airportLongitude],
func (db *DB) Get(locodeRecord locode.Record) (*locodedb.AirportRecord, error) {
if err := db.initAirports(); err != nil {
return nil, err
}
if related, err := db.isRelated(locodeRecord, airportRecord); err != nil || !related {
return err
records := db.mAirports[locodeRecord.LOCODE.CountryCode()]
for i := range records {
if locodeRecord.LOCODE.LocationCode() != records[i].iata &&
locodeRecord.NameWoDiacritics != records[i].city {
continue
}
lat, err := strconv.ParseFloat(airportRecord.lat, 64)
lat, err := strconv.ParseFloat(records[i].lat, 64)
if err != nil {
return err
return nil, err
}
lng, err := strconv.ParseFloat(airportRecord.lng, 64)
lng, err := strconv.ParseFloat(records[i].lng, 64)
if err != nil {
return err
return nil, err
}
rec = &locodedb.AirportRecord{
CountryName: airportRecord.country,
return &locodedb.AirportRecord{
CountryName: records[i].country,
Point: locodedb.NewPoint(lat, lng),
}, nil
}
return errScanInt
})
if err == nil && rec == nil {
err = locodedb.ErrAirportNotFound
}
return
}
func (db *DB) isRelated(locodeRecord locode.Record, airportRecord record) (bool, error) {
countryCode, err := db.countryCode(airportRecord.country)
if err != nil {
return false, errors.Wrap(err, "could not read country code of airport")
}
sameCountry := locodeRecord.LOCODE.CountryCode() == countryCode
sameIATA := locodeRecord.LOCODE.LocationCode() == airportRecord.iata
if sameCountry && sameIATA {
return true, nil
}
sameCity := locodeRecord.Name == airportRecord.city
return sameCountry && sameCity, nil
return nil, locodedb.ErrAirportNotFound
}
const (
@ -114,36 +89,68 @@ const (
countryFldNum
)
// CountryName scans records of the OpenFlights Country table, and returns
// name of the country by code.
// CountryName scans records of the OpenFlights Country table to in-memory table (once),
// and returns name of the country by code.
//
// Returns locodedb.ErrCountryNotFound if no entry matches.
func (db *DB) CountryName(code *locodedb.CountryCode) (name string, err error) {
err = db.scanWords(db.countries, countryFldNum, func(words []string) error {
if words[countryISOCode] == code.String() {
name = words[countryName]
return errScanInt
if err = db.initCountries(); err != nil {
return
}
return nil
})
argCode := code.String()
if err == nil && name == "" {
for cName, cCode := range db.mCountries {
if cCode == argCode {
name = cName
break
}
}
if name == "" {
err = locodedb.ErrCountryNotFound
}
return
}
func (db *DB) countryCode(country string) (code string, err error) {
err = db.scanWords(db.countries, countryFldNum, func(words []string) error {
if words[countryName] == country {
code = words[countryISOCode]
return errScanInt
func (db *DB) initAirports() (err error) {
db.airportsOnce.Do(func() {
db.mAirports = make(map[string][]record)
if err = db.initCountries(); err != nil {
return
}
err = db.scanWords(db.airports, airportFldNum, func(words []string) error {
countryCode := db.mCountries[words[airportCountry]]
if countryCode != "" {
db.mAirports[countryCode] = append(db.mAirports[countryCode], record{
city: words[airportCity],
country: words[airportCountry],
iata: words[airportIATA],
lat: words[airportLatitude],
lng: words[airportLongitude],
})
}
return nil
})
})
return
}
func (db *DB) initCountries() (err error) {
db.countriesOnce.Do(func() {
db.mCountries = make(map[string]string)
err = db.scanWords(db.countries, countryFldNum, func(words []string) error {
db.mCountries[words[countryName]] = words[countryISOCode]
return nil
})
})
return
}

View file

@ -3,6 +3,7 @@ package airportsdb
import (
"fmt"
"os"
"sync"
)
// Prm groups the required parameters of the DB's constructor.
@ -30,6 +31,12 @@ type Prm struct {
// The DB is immediately ready to work through API.
type DB struct {
airports, countries pathMode
airportsOnce, countriesOnce sync.Once
mCountries map[string]string
mAirports map[string][]record
}
type pathMode struct {