TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 100 Pull requests 6 Projects Releases 1 Activity Actions

[#1259] Remove locode package and add frostfs-locode-db dependency #1260

Merged
fyrchik merged 1 commit from george.bartolomey/frostfs-node:locode into master 2024-09-04 19:51:10 +00:00
Conversation 8 Commits 1 Files changed 30 +9 -1985
30 changed files with 9 additions and 1985 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
go.mod
View file

@ -7,6 +7,7 @@ require (
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0
git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de
git.frostfs.info/TrueCloudLab/hrw v1.2.1
@ -26,7 +27,6 @@ require (
github.com/nspcc-dev/neo-go v0.106.0
github.com/olekukonko/tablewriter v0.0.5
github.com/panjf2000/ants/v2 v2.9.0
github.com/paulmach/orb v0.11.0
github.com/prometheus/client_golang v1.19.0
github.com/spf13/cast v1.6.0
github.com/spf13/cobra v1.8.1
fyrchik commented 2024-07-22 12:43:07 +00:00
Outdated
Review
Copy link

Speaking of other dependencies too -- do you have any specific reason for update?

Speaking of other dependencies too -- do you have any specific reason for update?
george.bartolomey commented 2024-07-22 14:09:59 +00:00
Outdated
Review
Copy link

go mod tidy changes version of these dependencies.

frostfs-locode-db has newer version of packages. Could we downgrade these packages in frostfs-locode-db module?

`go mod tidy` changes version of these dependencies. frostfs-locode-db has newer version of packages. Could we downgrade these packages in frostfs-locode-db module?
fyrchik commented 2024-07-22 15:28:20 +00:00
Outdated
Review
Copy link

ok, not that important, let's leave as is.

ok, not that important, let's leave as is.
👍 1
fyrchik commented 2024-07-22 15:28:50 +00:00
Outdated
Review
Copy link

I will update in another PR, let's rebase this one after.

I will update in another PR, let's rebase this one after.
@ -51,7 +51,6 @@ require (
github.com/sagikazarmark/locafero v0.6.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
go.mongodb.org/mongo-driver v1.13.1 // indirect
)
require (

BIN
go.sum
View file

Binary file not shown.

6
pkg/innerring/locode.go
View file

@ -1,11 +1,11 @@
package innerring
import (
"git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode"
locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db"
locodebolt "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db/boltdb"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap"
irlocode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/locode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db"
locodebolt "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db/boltdb"
"github.com/spf13/viper"
)

2
pkg/innerring/processors/netmap/nodevalidation/locode/calls.go
View file

@ -4,7 +4,7 @@ import (
"errors"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
"git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
)

4
pkg/innerring/processors/netmap/nodevalidation/locode/calls_test.go
View file

@ -5,9 +5,9 @@ import (
"fmt"
"testing"
locodestd "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode"
locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/locode"
locodestd "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"github.com/stretchr/testify/require"
)

4
pkg/innerring/processors/netmap/nodevalidation/locode/deps.go
View file

@ -1,8 +1,8 @@
package locode
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db"
"git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode"
locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db"
)
// Record is an interface of read-only

193
pkg/util/locode/column/coordinates.go
View file

@ -1,193 +0,0 @@
package locodecolumn
import (
"fmt"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
)
const (
minutesDigits = 2
hemisphereSymbols = 1
)
const (
latDegDigits = 2
lngDegDigits = 3
)
type coordinateCode struct {
degDigits int
value []uint8
}
// LongitudeCode represents the value of the longitude
// of the location conforming to UN/LOCODE specification.
type LongitudeCode coordinateCode
// LongitudeHemisphere represents the hemisphere of the earth
// // along the Greenwich meridian.
type LongitudeHemisphere [hemisphereSymbols]uint8
// LatitudeCode represents the value of the latitude
// of the location conforming to UN/LOCODE specification.
type LatitudeCode coordinateCode
// LatitudeHemisphere represents the hemisphere of the earth
// along the equator.
type LatitudeHemisphere [hemisphereSymbols]uint8
func coordinateFromString(s string, degDigits int, hemisphereAlphabet []uint8) (*coordinateCode, error) {
if len(s) != degDigits+minutesDigits+hemisphereSymbols {
return nil, locode.ErrInvalidString
}
for i := range s[:degDigits+minutesDigits] {
if !isDigit(s[i]) {
return nil, locode.ErrInvalidString
}
}
loop:
for _, sym := range s[degDigits+minutesDigits:] {
for j := range hemisphereAlphabet {
if hemisphereAlphabet[j] == uint8(sym) {
continue loop
}
}
return nil, locode.ErrInvalidString
}
return &coordinateCode{
degDigits: degDigits,
value: []uint8(s),
}, nil
}
// LongitudeFromString parses a string and returns the location's longitude.
func LongitudeFromString(s string) (*LongitudeCode, error) {
cc, err := coordinateFromString(s, lngDegDigits, []uint8{'W', 'E'})
if err != nil {
return nil, err
}
return (*LongitudeCode)(cc), nil
}
// LatitudeFromString parses a string and returns the location's latitude.
func LatitudeFromString(s string) (*LatitudeCode, error) {
cc, err := coordinateFromString(s, latDegDigits, []uint8{'N', 'S'})
if err != nil {
return nil, err
}
return (*LatitudeCode)(cc), nil
}
func (cc *coordinateCode) degrees() []uint8 {
return cc.value[:cc.degDigits]
}
// Degrees returns the longitude's degrees.
func (lc *LongitudeCode) Degrees() (l [lngDegDigits]uint8) {
copy(l[:], (*coordinateCode)(lc).degrees())
return
}
// Degrees returns the latitude's degrees.
func (lc *LatitudeCode) Degrees() (l [latDegDigits]uint8) {
copy(l[:], (*coordinateCode)(lc).degrees())
return
}
func (cc *coordinateCode) minutes() (mnt [minutesDigits]uint8) {
for i := 0; i < minutesDigits; i++ {
mnt[i] = cc.value[cc.degDigits+i]
}
return
}
// Minutes returns the longitude's minutes.
func (lc *LongitudeCode) Minutes() [minutesDigits]uint8 {
return (*coordinateCode)(lc).minutes()
}
// Minutes returns the latitude's minutes.
func (lc *LatitudeCode) Minutes() [minutesDigits]uint8 {
return (*coordinateCode)(lc).minutes()
}
// Hemisphere returns the longitude's hemisphere code.
func (lc *LongitudeCode) Hemisphere() LongitudeHemisphere {
return (*coordinateCode)(lc).hemisphere()
}
// Hemisphere returns the latitude's hemisphere code.
func (lc *LatitudeCode) Hemisphere() LatitudeHemisphere {
return (*coordinateCode)(lc).hemisphere()
}
func (cc *coordinateCode) hemisphere() (h [hemisphereSymbols]uint8) {
for i := 0; i < hemisphereSymbols; i++ {
h[i] = cc.value[cc.degDigits+minutesDigits+i]
}
return h
}
// North returns true for the northern hemisphere.
func (h LatitudeHemisphere) North() bool {
return h[0] == 'N'
}
// East returns true for the eastern hemisphere.
func (h LongitudeHemisphere) East() bool {
return h[0] == 'E'
}
// Coordinates represents the coordinates of the location from UN/LOCODE table.
type Coordinates struct {
lat *LatitudeCode
lng *LongitudeCode
}
// Latitude returns the location's latitude.
func (c *Coordinates) Latitude() *LatitudeCode {
return c.lat
}
// Longitude returns the location's longitude.
func (c *Coordinates) Longitude() *LongitudeCode {
return c.lng
}
// CoordinatesFromString parses a string and returns the location's coordinates.
func CoordinatesFromString(s string) (*Coordinates, error) {
if len(s) == 0 {
return nil, nil
}
strs := strings.Split(s, " ")
if len(strs) != 2 {
return nil, locode.ErrInvalidString
}
lat, err := LatitudeFromString(strs[0])
if err != nil {
return nil, fmt.Errorf("could not parse latitude: %w", err)
}
lng, err := LongitudeFromString(strs[1])
if err != nil {
return nil, fmt.Errorf("could not parse longitude: %w", err)
}
return &Coordinates{
lat: lat,
lng: lng,
}, nil
}

38
pkg/util/locode/column/country.go
View file

@ -1,38 +0,0 @@
package locodecolumn
import (
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
)
const countryCodeLen = 2
// CountryCode represents ISO 3166 alpha-2 Country Code.
type CountryCode [countryCodeLen]uint8
// Symbols returns digits of the country code.
func (cc *CountryCode) Symbols() [countryCodeLen]uint8 {
return *cc
}
// CountryCodeFromString parses a string and returns the country code.
func CountryCodeFromString(s string) (*CountryCode, error) {
if l := len(s); l != countryCodeLen {
return nil, fmt.Errorf("incorrect country code length: expect: %d, got: %d",
countryCodeLen,
l,
)
}
for i := range s {
if !isUpperAlpha(s[i]) {
return nil, locode.ErrInvalidString
}
}
cc := CountryCode{}
copy(cc[:], s)
return &cc, nil
}

38
pkg/util/locode/column/location.go
View file

@ -1,38 +0,0 @@
package locodecolumn
import (
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
)
const locationCodeLen = 3
// LocationCode represents 3-character code for the location.
type LocationCode [locationCodeLen]uint8
// Symbols returns characters of the location code.
func (lc *LocationCode) Symbols() [locationCodeLen]uint8 {
return *lc
}
// LocationCodeFromString parses a string and returns the location code.
func LocationCodeFromString(s string) (*LocationCode, error) {
if l := len(s); l != locationCodeLen {
return nil, fmt.Errorf("incorrect location code length: expect: %d, got: %d",
locationCodeLen,
l,
)
}
for i := range s {
if !isUpperAlpha(s[i]) && !isDigit(s[i]) {
return nil, locode.ErrInvalidString
}
}
lc := LocationCode{}
copy(lc[:], s)
return &lc, nil
}

9
pkg/util/locode/column/util.go
View file

@ -1,9 +0,0 @@
package locodecolumn
func isDigit(sym uint8) bool {
return sym >= '0' && sym <= '9'
}
func isUpperAlpha(sym uint8) bool {
return sym >= 'A' && sym <= 'Z'
}

194
pkg/util/locode/db/airports/calls.go
View file

@ -1,194 +0,0 @@
package airportsdb
import (
"encoding/csv"
"errors"
"fmt"
"io"
"os"
"strconv"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db"
)
const (
_ = iota - 1
_ // Airport ID
_ // Name
airportCity
airportCountry
airportIATA
_ // ICAO
airportLatitude
airportLongitude
_ // Altitude
_ // Timezone
_ // DST
_ // Tz database time zone
_ // Type
_ // Source
airportFldNum
)
type record struct {
city,
country,
iata,
lat,
lng string
}
// Get scans the records of the OpenFlights Airport to an in-memory table (once),
// and returns an entry that matches the passed UN/LOCODE record.
//
// Records are matched if they have the same country code and either
// same IATA code or same city name (location name in UN/LOCODE).
//
// Returns locodedb.ErrAirportNotFound if no entry matches.
func (db *DB) Get(locodeRecord locode.Record) (*locodedb.AirportRecord, error) {
if err := db.initAirports(); err != nil {
return nil, err
}
records := db.mAirports[locodeRecord.LOCODE.CountryCode()]
for i := range records {
if locodeRecord.LOCODE.LocationCode() != records[i].iata &&
locodeRecord.NameWoDiacritics != records[i].city {
continue
}
lat, err := strconv.ParseFloat(records[i].lat, 64)
if err != nil {
return nil, err
}
lng, err := strconv.ParseFloat(records[i].lng, 64)
if err != nil {
return nil, err
}
return &locodedb.AirportRecord{
CountryName: records[i].country,
Point: locodedb.NewPoint(lat, lng),
}, nil
}
return nil, locodedb.ErrAirportNotFound
}
const (
_ = iota - 1
countryName
countryISOCode
_ // dafif_code
countryFldNum
)
// CountryName scans the records of the OpenFlights Country table to an in-memory table (once),
// and returns the name of the country by code.
//
// Returns locodedb.ErrCountryNotFound if no entry matches.
func (db *DB) CountryName(code *locodedb.CountryCode) (name string, err error) {
if err = db.initCountries(); err != nil {
return
}
argCode := code.String()
for cName, cCode := range db.mCountries {
if cCode == argCode {
name = cName
break
}
}
if name == "" {
err = locodedb.ErrCountryNotFound
}
return
}
func (db *DB) initAirports() (err error) {
db.airportsOnce.Do(func() {
db.mAirports = make(map[string][]record)
if err = db.initCountries(); err != nil {
return
}
err = db.scanWords(db.airports, airportFldNum, func(words []string) error {
countryCode := db.mCountries[words[airportCountry]]
if countryCode != "" {
db.mAirports[countryCode] = append(db.mAirports[countryCode], record{
city: words[airportCity],
country: words[airportCountry],
iata: words[airportIATA],
lat: words[airportLatitude],
lng: words[airportLongitude],
})
}
return nil
})
})
return
}
func (db *DB) initCountries() (err error) {
db.countriesOnce.Do(func() {
db.mCountries = make(map[string]string)
err = db.scanWords(db.countries, countryFldNum, func(words []string) error {
db.mCountries[words[countryName]] = words[countryISOCode]
return nil
})
})
return
}
var errScanInt = errors.New("interrupt scan")
func (db *DB) scanWords(pm pathMode, num int, wordsHandler func([]string) error) error {
tableFile, err := os.OpenFile(pm.path, os.O_RDONLY, pm.mode)
if err != nil {
return err
}
defer tableFile.Close()
r := csv.NewReader(tableFile)
r.ReuseRecord = true
for {
words, err := r.Read()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return err
} else if ln := len(words); ln != num {
return fmt.Errorf("unexpected number of words %d", ln)
}
if err := wordsHandler(words); err != nil {
if errors.Is(err, errScanInt) {
break
}
return err
}
}
return nil
}

83
pkg/util/locode/db/airports/db.go
View file

@ -1,83 +0,0 @@
package airportsdb
import (
"fmt"
"io/fs"
"sync"
)
// Prm groups the required parameters of the DB's constructor.
//
// All values must comply with the requirements imposed on them.
// Passing incorrect parameter values will result in constructor
// failure (error or panic depending on the implementation).
type Prm struct {
// Path to OpenFlights Airport csv table.
//
// Must not be empty.
AirportsPath string
// Path to OpenFlights Countries csv table.
//
// Must not be empty.
CountriesPath string
}
// DB is a descriptor of the OpenFlights database in csv format.
//
// For correct operation, DB must be created
// using the constructor (New) based on the required parameters
// and optional components. After successful creation,
// The DB is immediately ready to work through API.
type DB struct {
airports, countries pathMode
airportsOnce, countriesOnce sync.Once
mCountries map[string]string
mAirports map[string][]record
}
type pathMode struct {
path string
mode fs.FileMode
}
const invalidPrmValFmt = "invalid parameter %s (%T):%v"
func panicOnPrmValue(n string, v any) {
panic(fmt.Sprintf(invalidPrmValFmt, n, v, v))
}
// New creates a new instance of the DB.
//
// Panics if at least one value of the parameters is invalid.
//
// The created DB does not require additional
// initialization and is completely ready for work.
func New(prm Prm, opts ...Option) *DB {
switch {
case prm.AirportsPath == "":
panicOnPrmValue("AirportsPath", prm.AirportsPath)
case prm.CountriesPath == "":
panicOnPrmValue("CountriesPath", prm.CountriesPath)
}
o := defaultOpts()
for i := range opts {
opts[i](o)
}
return &DB{
airports: pathMode{
path: prm.AirportsPath,
mode: o.airportMode,
},
countries: pathMode{
path: prm.CountriesPath,
mode: o.countryMode,
},
}
}

19
pkg/util/locode/db/airports/opts.go
View file

@ -1,19 +0,0 @@
package airportsdb
import (
"io/fs"
)
// Option sets an optional parameter of DB.
type Option func(*options)
type options struct {
airportMode, countryMode fs.FileMode
}
func defaultOpts() *options {
return &options{
airportMode: fs.ModePerm, // 0777
countryMode: fs.ModePerm, // 0777
}
}

166
pkg/util/locode/db/boltdb/calls.go
View file

@ -1,166 +0,0 @@
package locodebolt
import (
"encoding/json"
"errors"
"fmt"
"path/filepath"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db"
"go.etcd.io/bbolt"
)
// Open opens an underlying BoltDB instance.
//
// Timeout of BoltDB opening is 3s (only for Linux or Darwin).
//
// Opens BoltDB in read-only mode if DB is read-only.
func (db *DB) Open() error {
// copy-paste from metabase:
// consider universal Open/Close for BoltDB wrappers
err := util.MkdirAllX(filepath.Dir(db.path), db.mode)
if err != nil {
return fmt.Errorf("could not create dir for BoltDB: %w", err)
}
db.bolt, err = bbolt.Open(db.path, db.mode, db.boltOpts)
if err != nil {
return fmt.Errorf("could not open BoltDB: %w", err)
}
return nil
}
// Close closes an underlying BoltDB instance.
//
// Must not be called before successful Open call.
func (db *DB) Close() error {
return db.bolt.Close()
}
func countryBucketKey(cc *locodedb.CountryCode) ([]byte, error) {
return []byte(cc.String()), nil
}
func locationBucketKey(lc *locodedb.LocationCode) ([]byte, error) {
return []byte(lc.String()), nil
}
type recordJSON struct {
CountryName string
LocationName string
SubDivName string
SubDivCode string
Latitude float64
Longitude float64
Continent string
}
func recordValue(r locodedb.Record) ([]byte, error) {
p := r.GeoPoint()
rj := &recordJSON{
CountryName: r.CountryName(),
LocationName: r.LocationName(),
SubDivName: r.SubDivName(),
SubDivCode: r.SubDivCode(),
Latitude: p.Latitude(),
Longitude: p.Longitude(),
Continent: r.Continent().String(),
}
return json.Marshal(rj)
}
func recordFromValue(data []byte) (*locodedb.Record, error) {
rj := new(recordJSON)
if err := json.Unmarshal(data, rj); err != nil {
return nil, err
}
r := new(locodedb.Record)
r.SetCountryName(rj.CountryName)
r.SetLocationName(rj.LocationName)
r.SetSubDivName(rj.SubDivName)
r.SetSubDivCode(rj.SubDivCode)
r.SetGeoPoint(locodedb.NewPoint(rj.Latitude, rj.Longitude))
cont := locodedb.ContinentFromString(rj.Continent)
r.SetContinent(&cont)
return r, nil
}
// Put saves the record by key in an underlying BoltDB instance.
//
// Country code from the key is used for allocating the 1st level buckets.
// Records are stored in country buckets by the location code from the key.
// The records are stored in internal binary JSON format.
//
// Must not be called before successful Open call.
// Must not be called in read-only mode: behavior is undefined.
func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error {
return db.bolt.Batch(func(tx *bbolt.Tx) error {
countryKey, err := countryBucketKey(key.CountryCode())
if err != nil {
return err
}
bktCountry, err := tx.CreateBucketIfNotExists(countryKey)
if err != nil {
return fmt.Errorf("could not create country bucket: %w", err)
}
locationKey, err := locationBucketKey(key.LocationCode())
if err != nil {
return err
}
cont, err := recordValue(rec)
if err != nil {
return err
}
return bktCountry.Put(locationKey, cont)
})
}
var errRecordNotFound = errors.New("record not found")
// Get reads the record by key from underlying BoltDB instance.
//
// Returns an error if no record is presented by key in DB.
//
// Must not be called before successful Open call.
func (db *DB) Get(key locodedb.Key) (rec *locodedb.Record, err error) {
err = db.bolt.View(func(tx *bbolt.Tx) error {
countryKey, err := countryBucketKey(key.CountryCode())
if err != nil {
return err
}
bktCountry := tx.Bucket(countryKey)
if bktCountry == nil {
return errRecordNotFound
}
locationKey, err := locationBucketKey(key.LocationCode())
if err != nil {
return err
}
data := bktCountry.Get(locationKey)
if data == nil {
return errRecordNotFound
}
rec, err = recordFromValue(data)
return err
})
return
}

73
pkg/util/locode/db/boltdb/db.go
View file

@ -1,73 +0,0 @@
package locodebolt
import (
"fmt"
"io/fs"
"go.etcd.io/bbolt"
)
// Prm groups the required parameters of the DB's constructor.
//
// All values must comply with the requirements imposed on them.
// Passing incorrect parameter values will result in constructor
// failure (error or panic depending on the implementation).
type Prm struct {
// Path to BoltDB file with FrostFS location database.
//
// Must not be empty.
Path string
}
// DB is a descriptor of the FrostFS BoltDB location database.
//
// For correct operation, DB must be created
// using the constructor (New) based on the required parameters
// and optional components.
//
// After successful creation,
// DB must be opened through Open call. After successful opening,
// DB is ready to work through API (until Close call).
//
// Upon completion of work with the DB, it must be closed
// by Close method.
type DB struct {
path string
mode fs.FileMode
boltOpts *bbolt.Options
bolt *bbolt.DB
}
const invalidPrmValFmt = "invalid parameter %s (%T):%v"
func panicOnPrmValue(n string, v any) {
panic(fmt.Sprintf(invalidPrmValFmt, n, v, v))
}
// New creates a new instance of the DB.
//
// Panics if at least one value of the parameters is invalid.
//
// The created DB requires calling the Open method in order
// to initialize required resources.
func New(prm Prm, opts ...Option) *DB {
switch {
case prm.Path == "":
panicOnPrmValue("Path", prm.Path)
}
o := defaultOpts()
for i := range opts {
opts[i](o)
}
return &DB{
path: prm.Path,
mode: o.mode,
boltOpts: o.boltOpts,
}
}

37
pkg/util/locode/db/boltdb/opts.go
View file

@ -1,37 +0,0 @@
package locodebolt
import (
"io/fs"
"os"
"time"
"go.etcd.io/bbolt"
)
// Option sets an optional parameter of DB.
type Option func(*options)
type options struct {
mode fs.FileMode
boltOpts *bbolt.Options
}
func defaultOpts() *options {
return &options{
mode: os.ModePerm, // 0777
boltOpts: &bbolt.Options{
Timeout: 3 * time.Second,
},
}
}
// ReadOnly enables read-only mode of the DB.
//
// Do not call DB.Put method on instances with
// this option: the behavior is undefined.
func ReadOnly() Option {
return func(o *options) {
o.boltOpts.ReadOnly = true
}
}

81
pkg/util/locode/db/continent.go
View file

@ -1,81 +0,0 @@
package locodedb
// Continent is an enumeration of Earth's continent.
type Continent uint8
const (
// ContinentUnknown is an undefined Continent value.
ContinentUnknown = iota
// ContinentEurope corresponds to Europe.
ContinentEurope
// ContinentAfrica corresponds to Africa.
ContinentAfrica
// ContinentNorthAmerica corresponds to North America.
ContinentNorthAmerica
// ContinentSouthAmerica corresponds to South America.
ContinentSouthAmerica
// ContinentAsia corresponds to Asia.
ContinentAsia
// ContinentAntarctica corresponds to Antarctica.
ContinentAntarctica
// ContinentOceania corresponds to Oceania.
ContinentOceania
)
// Is checks if c is the same continent as c2.
func (c *Continent) Is(c2 Continent) bool {
return *c == c2
}
func (c Continent) String() string {
switch c {
case ContinentUnknown:
fallthrough
default:
return "Unknown"
case ContinentEurope:
return "Europe"
case ContinentAfrica:
return "Africa"
case ContinentNorthAmerica:
return "North America"
case ContinentSouthAmerica:
return "South America"
case ContinentAsia:
return "Asia"
case ContinentAntarctica:
return "Antarctica"
case ContinentOceania:
return "Oceania"
}
}
// ContinentFromString returns Continent value
// corresponding to the passed string representation.
func ContinentFromString(str string) Continent {
switch str {
default:
return ContinentUnknown
case "Europe":
return ContinentEurope
case "Africa":
return ContinentAfrica
case "North America":
return ContinentNorthAmerica
case "South America":
return ContinentSouthAmerica
case "Asia":
return ContinentAsia
case "Antarctica":
return ContinentAntarctica
case "Oceania":
return ContinentOceania
}
}

98
pkg/util/locode/db/continents/geojson/calls.go
View file

@ -1,98 +0,0 @@
package continentsdb
import (
"fmt"
"os"
locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db"
"github.com/paulmach/orb"
"github.com/paulmach/orb/geojson"
"github.com/paulmach/orb/planar"
)
const continentProperty = "Continent"
// PointContinent goes through all polygons and returns the continent
// in which the point is located.
//
// Returns locodedb.ContinentUnknown if no entry matches.
//
// All GeoJSON feature are parsed from file once and stored in memory.
func (db *DB) PointContinent(point *locodedb.Point) (*locodedb.Continent, error) {
var err error
db.once.Do(func() {
err = db.init()
})
if err != nil {
return nil, err
}
planarPoint := orb.Point{point.Longitude(), point.Latitude()}
var (
continent string
minDst float64
)
for _, feature := range db.features {
if multiPolygon, ok := feature.Geometry.(orb.MultiPolygon); ok {
if planar.MultiPolygonContains(multiPolygon, planarPoint) {
continent = feature.Properties.MustString(continentProperty)
break
}
} else if polygon, ok := feature.Geometry.(orb.Polygon); ok {
if planar.PolygonContains(polygon, planarPoint) {
continent = feature.Properties.MustString(continentProperty)
break
}
}
distance := planar.DistanceFrom(feature.Geometry, planarPoint)
if minDst == 0 || minDst > distance {
minDst = distance
continent = feature.Properties.MustString(continentProperty)
}
}
c := continentFromString(continent)
return &c, nil
}
func (db *DB) init() error {
data, err := os.ReadFile(db.path)
if err != nil {
return fmt.Errorf("could not read data file: %w", err)
}
features, err := geojson.UnmarshalFeatureCollection(data)
if err != nil {
return fmt.Errorf("could not unmarshal GeoJSON feature collection: %w", err)
}
db.features = features.Features
return nil
}
func continentFromString(c string) locodedb.Continent {
switch c {
default:
return locodedb.ContinentUnknown
case "Africa":
return locodedb.ContinentAfrica
case "Asia":
return locodedb.ContinentAsia
case "Europe":
return locodedb.ContinentEurope
case "North America":
return locodedb.ContinentNorthAmerica
case "South America":
return locodedb.ContinentSouthAmerica
case "Antarctica":
return locodedb.ContinentAntarctica
case "Australia", "Oceania":
return locodedb.ContinentOceania
}
}

63
pkg/util/locode/db/continents/geojson/db.go
View file

@ -1,63 +0,0 @@
package continentsdb
import (
"fmt"
"sync"
"github.com/paulmach/orb/geojson"
)
// Prm groups the required parameters of the DB's constructor.
//
// All values must comply with the requirements imposed on them.
// Passing incorrect parameter values will result in constructor
// failure (error or panic depending on the implementation).
type Prm struct {
// Path to polygons of Earth's continents in GeoJSON format.
//
// Must not be empty.
Path string
}
// DB is a descriptor of the Earth's polygons in GeoJSON format.
//
// For correct operation, DB must be created
// using the constructor (New) based on the required parameters
// and optional components. After successful creation,
// The DB is immediately ready to work through API.
type DB struct {
path string
once sync.Once
features []*geojson.Feature
}
const invalidPrmValFmt = "invalid parameter %s (%T):%v"
func panicOnPrmValue(n string, v any) {
panic(fmt.Sprintf(invalidPrmValFmt, n, v, v))
}
// New creates a new instance of the DB.
//
// Panics if at least one value of the parameters is invalid.
//
// The created DB does not require additional
// initialization and is completely ready for work.
func New(prm Prm, opts ...Option) *DB {
switch {
case prm.Path == "":
panicOnPrmValue("Path", prm.Path)
}
o := defaultOpts()
for i := range opts {
opts[i](o)
}
return &DB{
path: prm.Path,
}
}

10
pkg/util/locode/db/continents/geojson/opts.go
View file

@ -1,10 +0,0 @@
package continentsdb
// Option sets an optional parameter of DB.
type Option func(*options)
type options struct{}
func defaultOpts() *options {
return &options{}
}

32
pkg/util/locode/db/country.go
View file

@ -1,32 +0,0 @@
package locodedb
import (
"fmt"
locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/column"
)
// CountryCode represents a country code for
// the storage in the FrostFS location database.
type CountryCode locodecolumn.CountryCode
// CountryCodeFromString parses a string UN/LOCODE country code
// and returns a CountryCode.
func CountryCodeFromString(s string) (*CountryCode, error) {
cc, err := locodecolumn.CountryCodeFromString(s)
if err != nil {
return nil, fmt.Errorf("could not parse country code: %w", err)
}
return CountryFromColumn(cc)
}
// CountryFromColumn converts a UN/LOCODE country code to a CountryCode.
func CountryFromColumn(cc *locodecolumn.CountryCode) (*CountryCode, error) {
return (*CountryCode)(cc), nil
}
func (c *CountryCode) String() string {
syms := (*locodecolumn.CountryCode)(c).Symbols()
return string(syms[:])
}

183
pkg/util/locode/db/db.go
View file

@ -1,183 +0,0 @@
package locodedb
import (
"errors"
"fmt"
"runtime"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
"golang.org/x/sync/errgroup"
)
// SourceTable is an interface of the UN/LOCODE table.
type SourceTable interface {
// Must iterate over all entries of the table
// and pass next entry to the handler.
//
// Must return handler's errors directly.
IterateAll(func(locode.Record) error) error
}
// DB is an interface of FrostFS location database.
type DB interface {
// Must save the record by key in the database.
Put(Key, Record) error
// Must return the record by key from the database.
Get(Key) (*Record, error)
}
// AirportRecord represents the entry in FrostFS airport database.
type AirportRecord struct {
// Name of the country where airport is located.
CountryName string
// Geo point where airport is located.
Point *Point
}
// ErrAirportNotFound is returned by AirportRecord readers
// when the required airport is not found.
var ErrAirportNotFound = errors.New("airport not found")
// AirportDB is an interface of FrostFS airport database.
type AirportDB interface {
// Must return the record by UN/LOCODE table record.
//
// Must return ErrAirportNotFound if there is no
// related airport in the database.
Get(locode.Record) (*AirportRecord, error)
}
// ContinentsDB is an interface of FrostFS continent database.
type ContinentsDB interface {
// Must return continent of the geo point.
PointContinent(*Point) (*Continent, error)
}
var ErrSubDivNotFound = errors.New("subdivision not found")
var ErrCountryNotFound = errors.New("country not found")
// NamesDB is an interface of the FrostFS location namespace.
type NamesDB interface {
// Must resolve a country code to a country name.
//
// Must return ErrCountryNotFound if there is no
// country with the provided code.
CountryName(*CountryCode) (string, error)
// Must resolve (country code, subdivision code) to
// a subdivision name.
//
// Must return ErrSubDivNotFound if either country or
// subdivision is not presented in database.
SubDivName(*CountryCode, string) (string, error)
}
// FillDatabase generates the FrostFS location database based on the UN/LOCODE table.
func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
var errG errgroup.Group
// Pick some sane default, after this the performance stopped increasing.
errG.SetLimit(runtime.NumCPU() * 4)
_ = table.IterateAll(func(tableRecord locode.Record) error {
errG.Go(func() error {
return processTableRecord(tableRecord, airports, continents, names, db)
})
return nil
})
return errG.Wait()
}
func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
if tableRecord.LOCODE.LocationCode() == "" {
return nil
}
dbKey, err := NewKey(tableRecord.LOCODE)
if err != nil {
return err
}
dbRecord, err := NewRecord(tableRecord)
if err != nil {
if errors.Is(err, errParseCoordinates) {
return nil
}
return err
}
geoPoint := dbRecord.GeoPoint()
countryName := ""
if geoPoint == nil {
airportRecord, err := airports.Get(tableRecord)
if err != nil {
if errors.Is(err, ErrAirportNotFound) {
return nil
}
return err
}
geoPoint = airportRecord.Point
countryName = airportRecord.CountryName
}
dbRecord.SetGeoPoint(geoPoint)
if countryName == "" {
countryName, err = names.CountryName(dbKey.CountryCode())
if err != nil {
if errors.Is(err, ErrCountryNotFound) {
return nil
}
return err
}
}
dbRecord.SetCountryName(countryName)
if subDivCode := dbRecord.SubDivCode(); subDivCode != "" {
subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode)
if err != nil {
if errors.Is(err, ErrSubDivNotFound) {
return nil
}
return err
}
dbRecord.SetSubDivName(subDivName)
}
continent, err := continents.PointContinent(geoPoint)
if err != nil {
return fmt.Errorf("could not calculate continent geo point: %w", err)
} else if continent.Is(ContinentUnknown) {
return nil
}
dbRecord.SetContinent(continent)
return db.Put(*dbKey, *dbRecord)
}
// LocodeRecord returns the record from the FrostFS location database
// corresponding to the string representation of UN/LOCODE.
func LocodeRecord(db DB, sLocode string) (*Record, error) {
lc, err := locode.FromString(sLocode)
if err != nil {
return nil, fmt.Errorf("could not parse locode: %w", err)
}
key, err := NewKey(*lc)
if err != nil {
return nil, err
}
return db.Get(*key)
}

32
pkg/util/locode/db/location.go
View file

@ -1,32 +0,0 @@
package locodedb
import (
"fmt"
locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/column"
)
// LocationCode represents a location code for
// the storage in the FrostFS location database.
type LocationCode locodecolumn.LocationCode
// LocationCodeFromString parses a string UN/LOCODE location code
// and returns a LocationCode.
func LocationCodeFromString(s string) (*LocationCode, error) {
lc, err := locodecolumn.LocationCodeFromString(s)
if err != nil {
return nil, fmt.Errorf("could not parse location code: %w", err)
}
return LocationFromColumn(lc)
}
// LocationFromColumn converts a UN/LOCODE country code to a LocationCode.
func LocationFromColumn(cc *locodecolumn.LocationCode) (*LocationCode, error) {
return (*LocationCode)(cc), nil
}
func (l *LocationCode) String() string {
syms := (*locodecolumn.LocationCode)(l).Symbols()
return string(syms[:])
}

93
pkg/util/locode/db/point.go
View file

@ -1,93 +0,0 @@
package locodedb
import (
"fmt"
"strconv"
locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/column"
)
// Point represents a 2D geographic point.
type Point struct {
lat, lng float64
}
// NewPoint creates, initializes and returns a new Point.
func NewPoint(lat, lng float64) *Point {
return &Point{
lat: lat,
lng: lng,
}
}
// Latitude returns the Point's latitude.
func (p Point) Latitude() float64 {
return p.lat
}
// Longitude returns the Point's longitude.
func (p Point) Longitude() float64 {
return p.lng
}
// PointFromCoordinates converts a UN/LOCODE coordinates to a Point.
func PointFromCoordinates(crd *locodecolumn.Coordinates) (*Point, error) {
if crd == nil {
return nil, nil
}
cLat := crd.Latitude()
cLatDeg := cLat.Degrees()
cLatMnt := cLat.Minutes()
lat, err := toDecimal(cLatDeg[:], cLatMnt[:])
if err != nil {
return nil, fmt.Errorf("could not parse latitude: %w", err)
}
if !cLat.Hemisphere().North() {
lat = -lat
}
cLng := crd.Longitude()
cLngDeg := cLng.Degrees()
cLngMnt := cLng.Minutes()
lng, err := toDecimal(cLngDeg[:], cLngMnt[:])
if err != nil {
return nil, fmt.Errorf("could not parse longitude: %w", err)
}
if !cLng.Hemisphere().East() {
lng = -lng
}
return &Point{
lat: lat,
lng: lng,
}, nil
}
func toDecimal(intRaw, minutesRaw []byte) (float64, error) {
integer, err := strconv.ParseFloat(string(intRaw), 64)
if err != nil {
return 0, fmt.Errorf("could not parse integer part: %w", err)
}
decimal, err := minutesToDegrees(minutesRaw)
if err != nil {
return 0, fmt.Errorf("could not parse decimal part: %w", err)
}
return integer + decimal, nil
}
// minutesToDegrees converts minutes to decimal part of a degree.
func minutesToDegrees(raw []byte) (float64, error) {
minutes, err := strconv.ParseFloat(string(raw), 64)
if err != nil {
return 0, err
}
return minutes / 60, nil
}

51
pkg/util/locode/db/point_test.go
View file

@ -1,51 +0,0 @@
package locodedb
import (
"testing"
locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/column"
"github.com/stretchr/testify/require"
)
func TestPointFromCoordinates(t *testing.T) {
testCases := []struct {
latGot, longGot string
latWant, longWant float64
}{
{
latGot: "5915N",
longGot: "01806E",
latWant: 59.25,
longWant: 18.10,
},
{
latGot: "1000N",
longGot: "02030E",
latWant: 10.00,
longWant: 20.50,
},
{
latGot: "0145S",
longGot: "03512W",
latWant: -01.75,
longWant: -35.20,
},
}
var (
crd *locodecolumn.Coordinates
point *Point
err error
)
for _, test := range testCases {
crd, err = locodecolumn.CoordinatesFromString(test.latGot + " " + test.longGot)
require.NoError(t, err)
point, err = PointFromCoordinates(crd)
require.NoError(t, err)
require.Equal(t, test.latWant, point.Latitude())
require.Equal(t, test.longWant, point.Longitude())
}
}

140
pkg/util/locode/db/record.go
View file

@ -1,140 +0,0 @@
package locodedb
import (
"errors"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/column"
)
// Key represents the key in FrostFS location database.
type Key struct {
cc *CountryCode
lc *LocationCode
}
// NewKey calculates Key from LOCODE.
func NewKey(lc locode.LOCODE) (*Key, error) {
country, err := CountryCodeFromString(lc.CountryCode())
if err != nil {
return nil, fmt.Errorf("could not parse country: %w", err)
}
location, err := LocationCodeFromString(lc.LocationCode())
if err != nil {
return nil, fmt.Errorf("could not parse location: %w", err)
}
return &Key{
cc: country,
lc: location,
}, nil
}
// CountryCode returns the location's country code.
func (k *Key) CountryCode() *CountryCode {
return k.cc
}
// LocationCode returns the location code.
func (k *Key) LocationCode() *LocationCode {
return k.lc
}
// Record represents the entry in FrostFS location database.
type Record struct {
countryName string
locationName string
subDivName string
subDivCode string
p *Point
cont *Continent
}
var errParseCoordinates = errors.New("invalid coordinates")
// NewRecord calculates the Record from the UN/LOCODE table record.
func NewRecord(r locode.Record) (*Record, error) {
crd, err := locodecolumn.CoordinatesFromString(r.Coordinates)
if err != nil {
return nil, fmt.Errorf("%w: %v", errParseCoordinates, err)
}
point, err := PointFromCoordinates(crd)
if err != nil {
return nil, fmt.Errorf("could not parse geo point: %w", err)
}
return &Record{
locationName: r.NameWoDiacritics,
subDivCode: r.SubDiv,
p: point,
}, nil
}
// CountryName returns the country name.
func (r *Record) CountryName() string {
return r.countryName
}
// SetCountryName sets the country name.
func (r *Record) SetCountryName(name string) {
r.countryName = name
}
// LocationName returns the location name.
func (r *Record) LocationName() string {
return r.locationName
}
// SetLocationName sets the location name.
func (r *Record) SetLocationName(name string) {
r.locationName = name
}
// SubDivCode returns the subdivision code.
func (r *Record) SubDivCode() string {
return r.subDivCode
}
// SetSubDivCode sets the subdivision code.
func (r *Record) SetSubDivCode(name string) {
r.subDivCode = name
}
// SubDivName returns the subdivision name.
func (r *Record) SubDivName() string {
return r.subDivName
}
// SetSubDivName sets the subdivision name.
func (r *Record) SetSubDivName(name string) {
r.subDivName = name
}
// GeoPoint returns geo point of the location.
func (r *Record) GeoPoint() *Point {
return r.p
}
// SetGeoPoint sets geo point of the location.
func (r *Record) SetGeoPoint(p *Point) {
r.p = p
}
// Continent returns the location continent.
func (r *Record) Continent() *Continent {
return r.cont
}
// SetContinent sets the location continent.
func (r *Record) SetContinent(c *Continent) {
r.cont = c
}

83
pkg/util/locode/record.go
View file

@ -1,83 +0,0 @@
package locode
import (
"errors"
"fmt"
"strings"
)
// LOCODE represents code from UN/LOCODE coding scheme.
type LOCODE [2]string
// Record represents a single record of the UN/LOCODE table.
type Record struct {
// Change Indicator.
Ch string
// Combination of a 2-character country code and a 3-character location code.
LOCODE LOCODE
// Name of the locations which has been allocated a UN/LOCODE.
Name string
// Names of the locations which have been allocated a UN/LOCODE without diacritic signs.
NameWoDiacritics string
// ISO 1-3 character alphabetic and/or numeric code for the administrative division of the country concerned.
SubDiv string
// 8-digit function classifier code for the location.
Function string
// Status of the entry by a 2-character code.
Status string
// Last date when the location was updated/entered.
Date string
// The IATA code for the location if different from location code in column LOCODE.
IATA string
// Geographical coordinates (latitude/longitude) of the location, if there is any.
Coordinates string
// Some general remarks regarding the UN/LOCODE in question.
Remarks string
}
// ErrInvalidString is the error of incorrect string format of the LOCODE.
var ErrInvalidString = errors.New("invalid string format in UN/Locode")
// FromString parses string and returns LOCODE.
//
// If string has incorrect format, ErrInvalidString returns.
func FromString(s string) (*LOCODE, error) {
const (
locationSeparator = " "
locodePartsNumber = 2
)
words := strings.Split(s, locationSeparator)
if ln := len(words); ln != locodePartsNumber {
return nil, fmt.Errorf(
"incorrect locode: it must consist of %d codes separated with a witespase, got: %d",
locodePartsNumber,
ln,
)
}
l := new(LOCODE)
copy(l[:], words)
return l, nil
}
// CountryCode returns a string representation of country code.
func (l *LOCODE) CountryCode() string {
return l[0]
}
// LocationCode returns a string representation of location code.
func (l *LOCODE) LocationCode() string {
return l[1]
}

156
pkg/util/locode/table/csv/calls.go
View file

@ -1,156 +0,0 @@
package csvlocode
import (
"encoding/csv"
"errors"
"io"
"os"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode"
locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db"
)
var errInvalidRecord = errors.New("invalid table record")
// IterateAll scans a table record one-by-one, parses a UN/LOCODE record
// from it and passes it to f.
//
// Returns f's errors directly.
func (t *Table) IterateAll(f func(locode.Record) error) error {
const wordsPerRecord = 12
return t.scanWords(t.paths, wordsPerRecord, func(words []string) error {
lc, err := locode.FromString(strings.Join(words[1:3], " "))
if err != nil {
return err
}
record := locode.Record{
Ch: words[0],
LOCODE: *lc,
Name: words[3],
NameWoDiacritics: words[4],
SubDiv: words[5],
Function: words[6],
Status: words[7],
Date: words[8],
IATA: words[9],
Coordinates: words[10],
Remarks: words[11],
}
return f(record)
})
}
const (
_ = iota - 1
subDivCountry
subDivSubdivision
subDivName
_ // subDivLevel
subDivFldNum
)
type subDivKey struct {
countryCode,
subDivCode string
}
type subDivRecord struct {
name string
}
// SubDivName scans a table record to an in-memory table (once),
// and returns the subdivision name of the country and the subdivision codes match.
//
// Returns locodedb.ErrSubDivNotFound if no entry matches.
func (t *Table) SubDivName(countryCode *locodedb.CountryCode, code string) (string, error) {
if err := t.initSubDiv(); err != nil {
return "", err
}
rec, ok := t.mSubDiv[subDivKey{
countryCode: countryCode.String(),
subDivCode: code,
}]
if !ok {
return "", locodedb.ErrSubDivNotFound
}
return rec.name, nil
}
func (t *Table) initSubDiv() (err error) {
t.subDivOnce.Do(func() {
t.mSubDiv = make(map[subDivKey]subDivRecord)
err = t.scanWords([]string{t.subDivPath}, subDivFldNum, func(words []string) error {
t.mSubDiv[subDivKey{
countryCode: words[subDivCountry],
subDivCode: words[subDivSubdivision],
}] = subDivRecord{
name: words[subDivName],
}
return nil
})
})
return
}
var errScanInt = errors.New("interrupt scan")
func (t *Table) scanWords(paths []string, fpr int, wordsHandler func([]string) error) error {
var (
rdrs = make([]io.Reader, 0, len(t.paths))
closers = make([]io.Closer, 0, len(t.paths))
)
for i := range paths {
file, err := os.OpenFile(paths[i], os.O_RDONLY, t.mode)
if err != nil {
return err
}
rdrs = append(rdrs, file)
closers = append(closers, file)
}
defer func() {
for i := range closers {
_ = closers[i].Close()
}
}()
r := csv.NewReader(io.MultiReader(rdrs...))
r.ReuseRecord = true
r.FieldsPerRecord = fpr
for {
words, err := r.Read()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return err
} else if len(words) != fpr {
return errInvalidRecord
}
if err := wordsHandler(words); err != nil {
if errors.Is(err, errScanInt) {
break
}
return err
}
}
return nil
}

28
pkg/util/locode/table/csv/opts.go
View file

@ -1,28 +0,0 @@
package csvlocode
import (
"io/fs"
)
// Option sets an optional parameter of Table.
type Option func(*options)
type options struct {
mode fs.FileMode
extraPaths []string
}
func defaultOpts() *options {
return &options{
mode: 0o700,
}
}
// WithExtraPaths returns an option to add extra paths
// to UN/LOCODE tables in csv format.
func WithExtraPaths(ps ...string) Option {
return func(o *options) {
o.extraPaths = append(o.extraPaths, ps...)
}
}

75
pkg/util/locode/table/csv/table.go
View file

@ -1,75 +0,0 @@
package csvlocode
import (
"fmt"
"io/fs"
"sync"
)
// Prm groups the required parameters of the Table's constructor.
//
// All values must comply with the requirements imposed on them.
// Passing incorrect parameter values will result in constructor
// failure (error or panic depending on the implementation).
type Prm struct {
// Path to UN/LOCODE csv table.
//
// Must not be empty.
Path string
// Path to csv table of UN/LOCODE Subdivisions.
//
// Must not be empty.
SubDivPath string
}
// Table is a descriptor of the UN/LOCODE table in csv format.
//
// For correct operation, Table must be created
// using the constructor (New) based on the required parameters
// and optional components. After successful creation,
// The Table is immediately ready to work through API.
type Table struct {
paths []string
mode fs.FileMode
subDivPath string
subDivOnce sync.Once
mSubDiv map[subDivKey]subDivRecord
}
const invalidPrmValFmt = "invalid parameter %s (%T):%v"
func panicOnPrmValue(n string, v any) {
panic(fmt.Sprintf(invalidPrmValFmt, n, v, v))
}
// New creates a new instance of the Table.
//
// Panics if at least one value of the parameters is invalid.
//
// The created Table does not require additional
// initialization and is completely ready for work.
func New(prm Prm, opts ...Option) *Table {
switch {
case prm.Path == "":
panicOnPrmValue("Path", prm.Path)
case prm.SubDivPath == "":
panicOnPrmValue("SubDivPath", prm.SubDivPath)
}
o := defaultOpts()
for i := range opts {
opts[i](o)
}
return &Table{
paths: append(o.extraPaths, prm.Path),
mode: o.mode,
subDivPath: prm.SubDivPath,
}
}