[#316] locode/airports: Scan csv table into memory

Scanning csv-table entries one-by-one takes significant time and system
resources. To speed up random access to table records, on the first call,
the table is pumped into memory (map). On subsequent calls, I/O operations
are not performed.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-02-10 19:52:02 +03:00 committed by Alex Vanin
parent 687c7d3b4a
commit 307355f165
2 changed files with 76 additions and 62 deletions

View file

@ -40,68 +40,43 @@ type record struct {
lng string lng string
} }
// Get scans records of the OpenFlights Airport table one-by-one, and // Get scans records of the OpenFlights Airport to in-memory table (once),
// returns entry that matches passed UN/LOCODE record. // and returns entry that matches passed UN/LOCODE record.
// //
// Records are matched if they have the same country code and either // Records are matched if they have the same country code and either
// same IATA code or same city name (location name in UN/LOCODE). // same IATA code or same city name (location name in UN/LOCODE).
// //
// Returns locodedb.ErrAirportNotFound if no entry matches. // Returns locodedb.ErrAirportNotFound if no entry matches.
func (db *DB) Get(locodeRecord locode.Record) (rec *locodedb.AirportRecord, err error) { func (db *DB) Get(locodeRecord locode.Record) (*locodedb.AirportRecord, error) {
err = db.scanWords(db.airports, airportFldNum, func(words []string) error { if err := db.initAirports(); err != nil {
airportRecord := record{ return nil, err
city: words[airportCity],
country: words[airportCountry],
iata: words[airportIATA],
lat: words[airportLatitude],
lng: words[airportLongitude],
} }
if related, err := db.isRelated(locodeRecord, airportRecord); err != nil || !related { records := db.mAirports[locodeRecord.LOCODE.CountryCode()]
return err
for i := range records {
if locodeRecord.LOCODE.LocationCode() != records[i].iata &&
locodeRecord.NameWoDiacritics != records[i].city {
continue
} }
lat, err := strconv.ParseFloat(airportRecord.lat, 64) lat, err := strconv.ParseFloat(records[i].lat, 64)
if err != nil { if err != nil {
return err return nil, err
} }
lng, err := strconv.ParseFloat(airportRecord.lng, 64) lng, err := strconv.ParseFloat(records[i].lng, 64)
if err != nil { if err != nil {
return err return nil, err
} }
rec = &locodedb.AirportRecord{ return &locodedb.AirportRecord{
CountryName: airportRecord.country, CountryName: records[i].country,
Point: locodedb.NewPoint(lat, lng), Point: locodedb.NewPoint(lat, lng),
}, nil
} }
return errScanInt return nil, locodedb.ErrAirportNotFound
})
if err == nil && rec == nil {
err = locodedb.ErrAirportNotFound
}
return
}
func (db *DB) isRelated(locodeRecord locode.Record, airportRecord record) (bool, error) {
countryCode, err := db.countryCode(airportRecord.country)
if err != nil {
return false, errors.Wrap(err, "could not read country code of airport")
}
sameCountry := locodeRecord.LOCODE.CountryCode() == countryCode
sameIATA := locodeRecord.LOCODE.LocationCode() == airportRecord.iata
if sameCountry && sameIATA {
return true, nil
}
sameCity := locodeRecord.Name == airportRecord.city
return sameCountry && sameCity, nil
} }
const ( const (
@ -114,36 +89,68 @@ const (
countryFldNum countryFldNum
) )
// CountryName scans records of the OpenFlights Country table, and returns // CountryName scans records of the OpenFlights Country table to in-memory table (once),
// name of the country by code. // and returns name of the country by code.
// //
// Returns locodedb.ErrCountryNotFound if no entry matches. // Returns locodedb.ErrCountryNotFound if no entry matches.
func (db *DB) CountryName(code *locodedb.CountryCode) (name string, err error) { func (db *DB) CountryName(code *locodedb.CountryCode) (name string, err error) {
err = db.scanWords(db.countries, countryFldNum, func(words []string) error { if err = db.initCountries(); err != nil {
if words[countryISOCode] == code.String() { return
name = words[countryName]
return errScanInt
} }
return nil argCode := code.String()
})
if err == nil && name == "" { for cName, cCode := range db.mCountries {
if cCode == argCode {
name = cName
break
}
}
if name == "" {
err = locodedb.ErrCountryNotFound err = locodedb.ErrCountryNotFound
} }
return return
} }
func (db *DB) countryCode(country string) (code string, err error) { func (db *DB) initAirports() (err error) {
err = db.scanWords(db.countries, countryFldNum, func(words []string) error { db.airportsOnce.Do(func() {
if words[countryName] == country { db.mAirports = make(map[string][]record)
code = words[countryISOCode]
return errScanInt if err = db.initCountries(); err != nil {
return
}
err = db.scanWords(db.airports, airportFldNum, func(words []string) error {
countryCode := db.mCountries[words[airportCountry]]
if countryCode != "" {
db.mAirports[countryCode] = append(db.mAirports[countryCode], record{
city: words[airportCity],
country: words[airportCountry],
iata: words[airportIATA],
lat: words[airportLatitude],
lng: words[airportLongitude],
})
} }
return nil return nil
}) })
})
return
}
func (db *DB) initCountries() (err error) {
db.countriesOnce.Do(func() {
db.mCountries = make(map[string]string)
err = db.scanWords(db.countries, countryFldNum, func(words []string) error {
db.mCountries[words[countryName]] = words[countryISOCode]
return nil
})
})
return return
} }

View file

@ -3,6 +3,7 @@ package airportsdb
import ( import (
"fmt" "fmt"
"os" "os"
"sync"
) )
// Prm groups the required parameters of the DB's constructor. // Prm groups the required parameters of the DB's constructor.
@ -30,6 +31,12 @@ type Prm struct {
// The DB is immediately ready to work through API. // The DB is immediately ready to work through API.
type DB struct { type DB struct {
airports, countries pathMode airports, countries pathMode
airportsOnce, countriesOnce sync.Once
mCountries map[string]string
mAirports map[string][]record
} }
type pathMode struct { type pathMode struct {