From 977229eb5aaed2dc2483a624e2299e649cf25058 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Wed, 10 Feb 2021 21:06:00 +0300 Subject: [PATCH] [#316] locode/csv: Scan subdivision csv-table into memory Scanning subdivision csv-table entries one-by-one takes significant time and system resources. To speed up random access to table records, on the first call, the table is pumped into memory (map). On subsequent calls, I/O operations are not performed. Signed-off-by: Leonard Lyubich --- pkg/util/locode/table/csv/calls.go | 68 +++++++++++++++++++++++------- pkg/util/locode/table/csv/table.go | 5 +++ 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/pkg/util/locode/table/csv/calls.go b/pkg/util/locode/table/csv/calls.go index 2a56723e9..77bce5b14 100644 --- a/pkg/util/locode/table/csv/calls.go +++ b/pkg/util/locode/table/csv/calls.go @@ -44,26 +44,62 @@ func (t *Table) IterateAll(f func(locode.Record) error) error { }) } -// SubDivName scans table record one-by-one, and returns subdivision name -// on country and subdivision codes match. +const ( + _ = iota - 1 + + subDivCountry + subDivSubdivision + subDivName + _ // subDivLevel + + subDivFldNum +) + +type subDivKey struct { + countryCode, + subDivCode string +} + +type subDivRecord struct { + name string +} + +// SubDivName scans table record to in-memory table (once), +// and returns subdivision name on country and subdivision codes match. // // Returns locodedb.ErrSubDivNotFound if no entry matches. -func (t *Table) SubDivName(countryCode *locodedb.CountryCode, name string) (subDiv string, err error) { - const wordsPerRecord = 4 - - err = t.scanWords([]string{t.subDivPath}, wordsPerRecord, func(words []string) error { - if words[0] == countryCode.String() && words[1] == name { - subDiv = words[2] - return errScanInt - } - - return nil - }) - - if err == nil && subDiv == "" { - err = locodedb.ErrSubDivNotFound +func (t *Table) SubDivName(countryCode *locodedb.CountryCode, code string) (string, error) { + if err := t.initSubDiv(); err != nil { + return "", err } + rec, ok := t.mSubDiv[subDivKey{ + countryCode: countryCode.String(), + subDivCode: code, + }] + if !ok { + return "", locodedb.ErrSubDivNotFound + } + + return rec.name, nil +} + +func (t *Table) initSubDiv() (err error) { + t.subDivOnce.Do(func() { + t.mSubDiv = make(map[subDivKey]subDivRecord) + + err = t.scanWords([]string{t.subDivPath}, subDivFldNum, func(words []string) error { + t.mSubDiv[subDivKey{ + countryCode: words[subDivCountry], + subDivCode: words[subDivSubdivision], + }] = subDivRecord{ + name: words[subDivName], + } + + return nil + }) + }) + return } diff --git a/pkg/util/locode/table/csv/table.go b/pkg/util/locode/table/csv/table.go index 43afd8428..e9bc3e207 100644 --- a/pkg/util/locode/table/csv/table.go +++ b/pkg/util/locode/table/csv/table.go @@ -3,6 +3,7 @@ package csvlocode import ( "fmt" "os" + "sync" ) // Prm groups the required parameters of the Table's constructor. @@ -34,6 +35,10 @@ type Table struct { mode os.FileMode subDivPath string + + subDivOnce sync.Once + + mSubDiv map[subDivKey]subDivRecord } const invalidPrmValFmt = "invalid parameter %s (%T):%v"