From 840b20538b9283c15bb637791324b0a8022f02d5 Mon Sep 17 00:00:00 2001 From: George Bartolomey Date: Mon, 8 Jul 2024 11:28:19 +0300 Subject: [PATCH] [#7] Add local tool for building database file Added frostfs-locode-db CLI utility that can generate and view UN/LOCODE database files. Go package git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode copied to this repository to eliminate interdependency between frostfs-node and frostfs-locode-db projects. The process of building database files reduced to starting make command. Signed-off-by: George Bartolomey --- Makefile | 19 ++- go.mod | 21 +++ go.sum | 110 ++++++++++++ locode_generate.go | 96 +++++++++++ locode_info.go | 55 ++++++ main.go | 45 +++++ pkg/locode/column/coordinates.go | 193 +++++++++++++++++++++ pkg/locode/column/country.go | 38 +++++ pkg/locode/column/location.go | 38 +++++ pkg/locode/column/util.go | 9 + pkg/locode/db/airports/calls.go | 194 ++++++++++++++++++++++ pkg/locode/db/airports/db.go | 83 +++++++++ pkg/locode/db/airports/opts.go | 19 +++ pkg/locode/db/boltdb/calls.go | 166 ++++++++++++++++++ pkg/locode/db/boltdb/db.go | 73 ++++++++ pkg/locode/db/boltdb/opts.go | 37 +++++ pkg/locode/db/continent.go | 81 +++++++++ pkg/locode/db/continents/geojson/calls.go | 98 +++++++++++ pkg/locode/db/continents/geojson/db.go | 63 +++++++ pkg/locode/db/continents/geojson/opts.go | 10 ++ pkg/locode/db/country.go | 32 ++++ pkg/locode/db/db.go | 183 ++++++++++++++++++++ pkg/locode/db/location.go | 32 ++++ pkg/locode/db/point.go | 93 +++++++++++ pkg/locode/db/point_test.go | 51 ++++++ pkg/locode/db/record.go | 140 ++++++++++++++++ pkg/locode/record.go | 83 +++++++++ pkg/locode/table/csv/calls.go | 156 +++++++++++++++++ pkg/locode/table/csv/opts.go | 28 ++++ pkg/locode/table/csv/table.go | 75 +++++++++ 30 files changed, 2317 insertions(+), 4 deletions(-) mode change 100644 => 100755 Makefile create mode 100644 go.mod create mode 100644 go.sum create mode 100644 locode_generate.go create mode 100644 locode_info.go create mode 100644 main.go create mode 100644 pkg/locode/column/coordinates.go create mode 100644 pkg/locode/column/country.go create mode 100644 pkg/locode/column/location.go create mode 100644 pkg/locode/column/util.go create mode 100644 pkg/locode/db/airports/calls.go create mode 100644 pkg/locode/db/airports/db.go create mode 100644 pkg/locode/db/airports/opts.go create mode 100644 pkg/locode/db/boltdb/calls.go create mode 100644 pkg/locode/db/boltdb/db.go create mode 100644 pkg/locode/db/boltdb/opts.go create mode 100644 pkg/locode/db/continent.go create mode 100644 pkg/locode/db/continents/geojson/calls.go create mode 100644 pkg/locode/db/continents/geojson/db.go create mode 100644 pkg/locode/db/continents/geojson/opts.go create mode 100644 pkg/locode/db/country.go create mode 100644 pkg/locode/db/db.go create mode 100644 pkg/locode/db/location.go create mode 100644 pkg/locode/db/point.go create mode 100644 pkg/locode/db/point_test.go create mode 100644 pkg/locode/db/record.go create mode 100644 pkg/locode/record.go create mode 100644 pkg/locode/table/csv/calls.go create mode 100644 pkg/locode/table/csv/opts.go create mode 100644 pkg/locode/table/csv/table.go diff --git a/Makefile b/Makefile old mode 100644 new mode 100755 index 1716325..522fb6c --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ #!/usr/bin/make -f VERSION ?= "$(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")" -FROST ?= frostfs-cli +FROST_LOCODE = ./frostfs-locode-db .PHONY: all clean version help update debpackage @@ -17,7 +17,10 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed "s/-/~/")-${OS_RELEASE} # Generate locode_db BoltDB file -all: $(DIRS) locode_db +all: $(DIRS) $(FROST_LOCODE) locode_db + +$(FROST_LOCODE): + go build . $(DIRS): @echo "⇒ Ensure dir: $@" @@ -39,8 +42,8 @@ in/unlocode-CodeList.csv: $(DIRS) zcat data/unlocode-CodeList.csv.gz > in/unlocode-CodeList.csv # Generate locode_db BoltDB file -locode_db: in/unlocode-CodeList.csv in/unlocode-SubdivisionCodes.csv in/continents.geojson in/airports.dat in/countries.dat - $(FROST) util locode generate \ +locode_db: in/unlocode-CodeList.csv in/unlocode-SubdivisionCodes.csv in/continents.geojson in/airports.dat in/countries.dat $(FROST_LOCODE) + $(FROST_LOCODE) generate \ --airports in/airports.dat \ --continents in/continents.geojson \ --countries in/countries.dat \ @@ -91,6 +94,13 @@ help: @echo '' @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u +# Run tests +test: locode_db + test -n "$$($(FROST_LOCODE) info --db locode_db --locode "US NYC" 2>&1 | grep "New York")" + test -n "$$($(FROST_LOCODE) info --db locode_db --locode "RU LED" 2>&1 | grep "Leningrad")" + test -n "$$($(FROST_LOCODE) info --db locode_db --locode "RU KUF" 2>&1 | grep "Samara")" + test -n "$$($(FROST_LOCODE) info --db locode_db --locode "VN DAN" 2>&1 | grep "Binh Hoa")" + test -n "$$($(FROST_LOCODE) info --db locode_db --locode "FR PAR" 2>&1 | grep "Paris")" # Clean data directory before update clean_data: @@ -104,6 +114,7 @@ clean: rm -f in/* rm -f tmp/* rm -f locode_db + rm -f $(FROST_LOCODE) # Package for Debian debpackage: diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..4b77381 --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module git.frostfs.info/TrueCloudLab/frostfs-locode-db + +go 1.21 + +require ( + github.com/paulmach/orb v0.11.1 + github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.9.0 + go.etcd.io/bbolt v1.3.10 + golang.org/x/sync v0.7.0 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.mongodb.org/mongo-driver v1.13.1 // indirect + golang.org/x/sys v0.18.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..74c70b4 --- /dev/null +++ b/go.sum @@ -0,0 +1,110 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= +go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/locode_generate.go b/locode_generate.go new file mode 100644 index 0000000..57ec4f8 --- /dev/null +++ b/locode_generate.go @@ -0,0 +1,96 @@ +package main + +import ( + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" + airportsdb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db/airports" + locodebolt "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db/boltdb" + continentsdb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db/continents/geojson" + csvlocode "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/table/csv" + "github.com/spf13/cobra" +) + +type namesDB struct { + *airportsdb.DB + *csvlocode.Table +} + +const ( + locodeGenerateInputFlag = "in" + locodeGenerateSubDivFlag = "subdiv" + locodeGenerateAirportsFlag = "airports" + locodeGenerateCountriesFlag = "countries" + locodeGenerateContinentsFlag = "continents" + locodeGenerateOutputFlag = "out" +) + +var ( + locodeGenerateInPaths []string + locodeGenerateSubDivPath string + locodeGenerateAirportsPath string + locodeGenerateCountriesPath string + locodeGenerateContinentsPath string + locodeGenerateOutPath string + + locodeGenerateCmd = &cobra.Command{ + Use: "generate", + Short: "Generate UN/LOCODE database for FrostFS", + Run: func(cmd *cobra.Command, _ []string) { + + locodeDB := csvlocode.New( + csvlocode.Prm{ + Path: locodeGenerateInPaths[0], + SubDivPath: locodeGenerateSubDivPath, + }, + csvlocode.WithExtraPaths(locodeGenerateInPaths[1:]...), + ) + + airportDB := airportsdb.New(airportsdb.Prm{ + AirportsPath: locodeGenerateAirportsPath, + CountriesPath: locodeGenerateCountriesPath, + }) + + continentsDB := continentsdb.New(continentsdb.Prm{ + Path: locodeGenerateContinentsPath, + }) + + targetDB := locodebolt.New(locodebolt.Prm{ + Path: locodeGenerateOutPath, + }) + + err := targetDB.Open() + ExitOnErr(cmd, "", err) + + defer targetDB.Close() + + names := &namesDB{ + DB: airportDB, + Table: locodeDB, + } + + err = locodedb.FillDatabase(locodeDB, airportDB, continentsDB, names, targetDB) + ExitOnErr(cmd, "", err) + }, + } +) + +func initUtilLocodeGenerateCmd() { + flags := locodeGenerateCmd.Flags() + + flags.StringSliceVar(&locodeGenerateInPaths, locodeGenerateInputFlag, nil, "List of paths to UN/LOCODE tables (csv)") + _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateInputFlag) + + flags.StringVar(&locodeGenerateSubDivPath, locodeGenerateSubDivFlag, "", "Path to UN/LOCODE subdivision database (csv)") + _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateSubDivFlag) + + flags.StringVar(&locodeGenerateAirportsPath, locodeGenerateAirportsFlag, "", "Path to OpenFlights airport database (csv)") + _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateAirportsFlag) + + flags.StringVar(&locodeGenerateCountriesPath, locodeGenerateCountriesFlag, "", "Path to OpenFlights country database (csv)") + _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateCountriesFlag) + + flags.StringVar(&locodeGenerateContinentsPath, locodeGenerateContinentsFlag, "", "Path to continent polygons (GeoJSON)") + _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateContinentsFlag) + + flags.StringVar(&locodeGenerateOutPath, locodeGenerateOutputFlag, "", "Target path for generated database") + _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateOutputFlag) +} diff --git a/locode_info.go b/locode_info.go new file mode 100644 index 0000000..6263a55 --- /dev/null +++ b/locode_info.go @@ -0,0 +1,55 @@ +package main + +import ( + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" + locodebolt "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db/boltdb" + "github.com/spf13/cobra" +) + +const ( + locodeInfoDBFlag = "db" + locodeInfoCodeFlag = "locode" +) + +var ( + locodeInfoDBPath string + locodeInfoCode string + + locodeInfoCmd = &cobra.Command{ + Use: "info", + Short: "Print information about UN/LOCODE from FrostFS database", + Run: func(cmd *cobra.Command, _ []string) { + targetDB := locodebolt.New(locodebolt.Prm{ + Path: locodeInfoDBPath, + }, locodebolt.ReadOnly()) + + err := targetDB.Open() + ExitOnErr(cmd, "", err) + + defer targetDB.Close() + + record, err := locodedb.LocodeRecord(targetDB, locodeInfoCode) + ExitOnErr(cmd, "", err) + + cmd.Printf("Country: %s\n", record.CountryName()) + cmd.Printf("Location: %s\n", record.LocationName()) + cmd.Printf("Continent: %s\n", record.Continent()) + if subDivCode := record.SubDivCode(); subDivCode != "" { + cmd.Printf("Subdivision: [%s] %s\n", subDivCode, record.SubDivName()) + } + + geoPoint := record.GeoPoint() + cmd.Printf("Coordinates: %0.2f, %0.2f\n", geoPoint.Latitude(), geoPoint.Longitude()) + }, + } +) + +func initUtilLocodeInfoCmd() { + flags := locodeInfoCmd.Flags() + + flags.StringVar(&locodeInfoDBPath, locodeInfoDBFlag, "", "Path to FrostFS UN/LOCODE database") + _ = locodeInfoCmd.MarkFlagRequired(locodeInfoDBFlag) + + flags.StringVar(&locodeInfoCode, locodeInfoCodeFlag, "", "UN/LOCODE") + _ = locodeInfoCmd.MarkFlagRequired(locodeInfoCodeFlag) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..f04cf17 --- /dev/null +++ b/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "frostfs-locode-db", + Short: "Command Line Tool to work with FrostFS' UN/LOCODE databases", + Long: "This tool can be used for generating or accessing FrostFS UN/LOCODE databases.", +} + +func ExitOnErr(cmd *cobra.Command, errFmt string, err error) { + if err == nil { + return + } + + if errFmt != "" { + err = fmt.Errorf(errFmt, err) + } + + const ( + _ = iota + internal + aclDenied + ) + cmd.PrintErrln(err) + if cmd.PersistentPostRun != nil { + cmd.PersistentPostRun(cmd, nil) + } + os.Exit(internal) +} + +func main() { + initUtilLocodeGenerateCmd() + initUtilLocodeInfoCmd() + rootCmd.AddCommand(locodeGenerateCmd, locodeInfoCmd) + err := rootCmd.Execute() + if err != nil { + ExitOnErr(rootCmd, "", err) + } +} diff --git a/pkg/locode/column/coordinates.go b/pkg/locode/column/coordinates.go new file mode 100644 index 0000000..66b0d47 --- /dev/null +++ b/pkg/locode/column/coordinates.go @@ -0,0 +1,193 @@ +package locodecolumn + +import ( + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" +) + +const ( + minutesDigits = 2 + hemisphereSymbols = 1 +) + +const ( + latDegDigits = 2 + lngDegDigits = 3 +) + +type coordinateCode struct { + degDigits int + value []uint8 +} + +// LongitudeCode represents the value of the longitude +// of the location conforming to UN/LOCODE specification. +type LongitudeCode coordinateCode + +// LongitudeHemisphere represents the hemisphere of the earth +// // along the Greenwich meridian. +type LongitudeHemisphere [hemisphereSymbols]uint8 + +// LatitudeCode represents the value of the latitude +// of the location conforming to UN/LOCODE specification. +type LatitudeCode coordinateCode + +// LatitudeHemisphere represents the hemisphere of the earth +// along the equator. +type LatitudeHemisphere [hemisphereSymbols]uint8 + +func coordinateFromString(s string, degDigits int, hemisphereAlphabet []uint8) (*coordinateCode, error) { + if len(s) != degDigits+minutesDigits+hemisphereSymbols { + return nil, locode.ErrInvalidString + } + + for i := range s[:degDigits+minutesDigits] { + if !isDigit(s[i]) { + return nil, locode.ErrInvalidString + } + } + +loop: + for _, sym := range s[degDigits+minutesDigits:] { + for j := range hemisphereAlphabet { + if hemisphereAlphabet[j] == uint8(sym) { + continue loop + } + } + + return nil, locode.ErrInvalidString + } + + return &coordinateCode{ + degDigits: degDigits, + value: []uint8(s), + }, nil +} + +// LongitudeFromString parses a string and returns the location's longitude. +func LongitudeFromString(s string) (*LongitudeCode, error) { + cc, err := coordinateFromString(s, lngDegDigits, []uint8{'W', 'E'}) + if err != nil { + return nil, err + } + + return (*LongitudeCode)(cc), nil +} + +// LatitudeFromString parses a string and returns the location's latitude. +func LatitudeFromString(s string) (*LatitudeCode, error) { + cc, err := coordinateFromString(s, latDegDigits, []uint8{'N', 'S'}) + if err != nil { + return nil, err + } + + return (*LatitudeCode)(cc), nil +} + +func (cc *coordinateCode) degrees() []uint8 { + return cc.value[:cc.degDigits] +} + +// Degrees returns the longitude's degrees. +func (lc *LongitudeCode) Degrees() (l [lngDegDigits]uint8) { + copy(l[:], (*coordinateCode)(lc).degrees()) + return +} + +// Degrees returns the latitude's degrees. +func (lc *LatitudeCode) Degrees() (l [latDegDigits]uint8) { + copy(l[:], (*coordinateCode)(lc).degrees()) + return +} + +func (cc *coordinateCode) minutes() (mnt [minutesDigits]uint8) { + for i := 0; i < minutesDigits; i++ { + mnt[i] = cc.value[cc.degDigits+i] + } + + return +} + +// Minutes returns the longitude's minutes. +func (lc *LongitudeCode) Minutes() [minutesDigits]uint8 { + return (*coordinateCode)(lc).minutes() +} + +// Minutes returns the latitude's minutes. +func (lc *LatitudeCode) Minutes() [minutesDigits]uint8 { + return (*coordinateCode)(lc).minutes() +} + +// Hemisphere returns the longitude's hemisphere code. +func (lc *LongitudeCode) Hemisphere() LongitudeHemisphere { + return (*coordinateCode)(lc).hemisphere() +} + +// Hemisphere returns the latitude's hemisphere code. +func (lc *LatitudeCode) Hemisphere() LatitudeHemisphere { + return (*coordinateCode)(lc).hemisphere() +} + +func (cc *coordinateCode) hemisphere() (h [hemisphereSymbols]uint8) { + for i := 0; i < hemisphereSymbols; i++ { + h[i] = cc.value[cc.degDigits+minutesDigits+i] + } + + return h +} + +// North returns true for the northern hemisphere. +func (h LatitudeHemisphere) North() bool { + return h[0] == 'N' +} + +// East returns true for the eastern hemisphere. +func (h LongitudeHemisphere) East() bool { + return h[0] == 'E' +} + +// Coordinates represents the coordinates of the location from UN/LOCODE table. +type Coordinates struct { + lat *LatitudeCode + + lng *LongitudeCode +} + +// Latitude returns the location's latitude. +func (c *Coordinates) Latitude() *LatitudeCode { + return c.lat +} + +// Longitude returns the location's longitude. +func (c *Coordinates) Longitude() *LongitudeCode { + return c.lng +} + +// CoordinatesFromString parses a string and returns the location's coordinates. +func CoordinatesFromString(s string) (*Coordinates, error) { + if len(s) == 0 { + return nil, nil + } + + strs := strings.Split(s, " ") + if len(strs) != 2 { + return nil, locode.ErrInvalidString + } + + lat, err := LatitudeFromString(strs[0]) + if err != nil { + return nil, fmt.Errorf("could not parse latitude: %w", err) + } + + lng, err := LongitudeFromString(strs[1]) + if err != nil { + return nil, fmt.Errorf("could not parse longitude: %w", err) + } + + return &Coordinates{ + lat: lat, + lng: lng, + }, nil +} diff --git a/pkg/locode/column/country.go b/pkg/locode/column/country.go new file mode 100644 index 0000000..2986a86 --- /dev/null +++ b/pkg/locode/column/country.go @@ -0,0 +1,38 @@ +package locodecolumn + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" +) + +const countryCodeLen = 2 + +// CountryCode represents ISO 3166 alpha-2 Country Code. +type CountryCode [countryCodeLen]uint8 + +// Symbols returns digits of the country code. +func (cc *CountryCode) Symbols() [countryCodeLen]uint8 { + return *cc +} + +// CountryCodeFromString parses a string and returns the country code. +func CountryCodeFromString(s string) (*CountryCode, error) { + if l := len(s); l != countryCodeLen { + return nil, fmt.Errorf("incorrect country code length: expect: %d, got: %d", + countryCodeLen, + l, + ) + } + + for i := range s { + if !isUpperAlpha(s[i]) { + return nil, locode.ErrInvalidString + } + } + + cc := CountryCode{} + copy(cc[:], s) + + return &cc, nil +} diff --git a/pkg/locode/column/location.go b/pkg/locode/column/location.go new file mode 100644 index 0000000..8a959bf --- /dev/null +++ b/pkg/locode/column/location.go @@ -0,0 +1,38 @@ +package locodecolumn + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" +) + +const locationCodeLen = 3 + +// LocationCode represents 3-character code for the location. +type LocationCode [locationCodeLen]uint8 + +// Symbols returns characters of the location code. +func (lc *LocationCode) Symbols() [locationCodeLen]uint8 { + return *lc +} + +// LocationCodeFromString parses a string and returns the location code. +func LocationCodeFromString(s string) (*LocationCode, error) { + if l := len(s); l != locationCodeLen { + return nil, fmt.Errorf("incorrect location code length: expect: %d, got: %d", + locationCodeLen, + l, + ) + } + + for i := range s { + if !isUpperAlpha(s[i]) && !isDigit(s[i]) { + return nil, locode.ErrInvalidString + } + } + + lc := LocationCode{} + copy(lc[:], s) + + return &lc, nil +} diff --git a/pkg/locode/column/util.go b/pkg/locode/column/util.go new file mode 100644 index 0000000..8da1f9a --- /dev/null +++ b/pkg/locode/column/util.go @@ -0,0 +1,9 @@ +package locodecolumn + +func isDigit(sym uint8) bool { + return sym >= '0' && sym <= '9' +} + +func isUpperAlpha(sym uint8) bool { + return sym >= 'A' && sym <= 'Z' +} diff --git a/pkg/locode/db/airports/calls.go b/pkg/locode/db/airports/calls.go new file mode 100644 index 0000000..3f7398b --- /dev/null +++ b/pkg/locode/db/airports/calls.go @@ -0,0 +1,194 @@ +package airportsdb + +import ( + "encoding/csv" + "errors" + "fmt" + "io" + "os" + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" +) + +const ( + _ = iota - 1 + + _ // Airport ID + _ // Name + airportCity + airportCountry + airportIATA + _ // ICAO + airportLatitude + airportLongitude + _ // Altitude + _ // Timezone + _ // DST + _ // Tz database time zone + _ // Type + _ // Source + + airportFldNum +) + +type record struct { + city, + country, + iata, + lat, + lng string +} + +// Get scans the records of the OpenFlights Airport to an in-memory table (once), +// and returns an entry that matches the passed UN/LOCODE record. +// +// Records are matched if they have the same country code and either +// same IATA code or same city name (location name in UN/LOCODE). +// +// Returns locodedb.ErrAirportNotFound if no entry matches. +func (db *DB) Get(locodeRecord locode.Record) (*locodedb.AirportRecord, error) { + if err := db.initAirports(); err != nil { + return nil, err + } + + records := db.mAirports[locodeRecord.LOCODE.CountryCode()] + + for i := range records { + if locodeRecord.LOCODE.LocationCode() != records[i].iata && + locodeRecord.NameWoDiacritics != records[i].city { + continue + } + + lat, err := strconv.ParseFloat(records[i].lat, 64) + if err != nil { + return nil, err + } + + lng, err := strconv.ParseFloat(records[i].lng, 64) + if err != nil { + return nil, err + } + + return &locodedb.AirportRecord{ + CountryName: records[i].country, + Point: locodedb.NewPoint(lat, lng), + }, nil + } + + return nil, locodedb.ErrAirportNotFound +} + +const ( + _ = iota - 1 + + countryName + countryISOCode + _ // dafif_code + + countryFldNum +) + +// CountryName scans the records of the OpenFlights Country table to an in-memory table (once), +// and returns the name of the country by code. +// +// Returns locodedb.ErrCountryNotFound if no entry matches. +func (db *DB) CountryName(code *locodedb.CountryCode) (name string, err error) { + if err = db.initCountries(); err != nil { + return + } + + argCode := code.String() + + for cName, cCode := range db.mCountries { + if cCode == argCode { + name = cName + break + } + } + + if name == "" { + err = locodedb.ErrCountryNotFound + } + + return +} + +func (db *DB) initAirports() (err error) { + db.airportsOnce.Do(func() { + db.mAirports = make(map[string][]record) + + if err = db.initCountries(); err != nil { + return + } + + err = db.scanWords(db.airports, airportFldNum, func(words []string) error { + countryCode := db.mCountries[words[airportCountry]] + if countryCode != "" { + db.mAirports[countryCode] = append(db.mAirports[countryCode], record{ + city: words[airportCity], + country: words[airportCountry], + iata: words[airportIATA], + lat: words[airportLatitude], + lng: words[airportLongitude], + }) + } + + return nil + }) + }) + + return +} + +func (db *DB) initCountries() (err error) { + db.countriesOnce.Do(func() { + db.mCountries = make(map[string]string) + + err = db.scanWords(db.countries, countryFldNum, func(words []string) error { + db.mCountries[words[countryName]] = words[countryISOCode] + + return nil + }) + }) + + return +} + +var errScanInt = errors.New("interrupt scan") + +func (db *DB) scanWords(pm pathMode, num int, wordsHandler func([]string) error) error { + tableFile, err := os.OpenFile(pm.path, os.O_RDONLY, pm.mode) + if err != nil { + return err + } + + defer tableFile.Close() + + r := csv.NewReader(tableFile) + r.ReuseRecord = true + + for { + words, err := r.Read() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + + return err + } else if ln := len(words); ln != num { + return fmt.Errorf("unexpected number of words %d", ln) + } + + if err := wordsHandler(words); err != nil { + if errors.Is(err, errScanInt) { + break + } + + return err + } + } + + return nil +} diff --git a/pkg/locode/db/airports/db.go b/pkg/locode/db/airports/db.go new file mode 100644 index 0000000..acfa3fd --- /dev/null +++ b/pkg/locode/db/airports/db.go @@ -0,0 +1,83 @@ +package airportsdb + +import ( + "fmt" + "io/fs" + "sync" +) + +// Prm groups the required parameters of the DB's constructor. +// +// All values must comply with the requirements imposed on them. +// Passing incorrect parameter values will result in constructor +// failure (error or panic depending on the implementation). +type Prm struct { + // Path to OpenFlights Airport csv table. + // + // Must not be empty. + AirportsPath string + + // Path to OpenFlights Countries csv table. + // + // Must not be empty. + CountriesPath string +} + +// DB is a descriptor of the OpenFlights database in csv format. +// +// For correct operation, DB must be created +// using the constructor (New) based on the required parameters +// and optional components. After successful creation, +// The DB is immediately ready to work through API. +type DB struct { + airports, countries pathMode + + airportsOnce, countriesOnce sync.Once + + mCountries map[string]string + + mAirports map[string][]record +} + +type pathMode struct { + path string + mode fs.FileMode +} + +const invalidPrmValFmt = "invalid parameter %s (%T):%v" + +func panicOnPrmValue(n string, v any) { + panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) +} + +// New creates a new instance of the DB. +// +// Panics if at least one value of the parameters is invalid. +// +// The created DB does not require additional +// initialization and is completely ready for work. +func New(prm Prm, opts ...Option) *DB { + switch { + case prm.AirportsPath == "": + panicOnPrmValue("AirportsPath", prm.AirportsPath) + case prm.CountriesPath == "": + panicOnPrmValue("CountriesPath", prm.CountriesPath) + } + + o := defaultOpts() + + for i := range opts { + opts[i](o) + } + + return &DB{ + airports: pathMode{ + path: prm.AirportsPath, + mode: o.airportMode, + }, + countries: pathMode{ + path: prm.CountriesPath, + mode: o.countryMode, + }, + } +} diff --git a/pkg/locode/db/airports/opts.go b/pkg/locode/db/airports/opts.go new file mode 100644 index 0000000..3799d9e --- /dev/null +++ b/pkg/locode/db/airports/opts.go @@ -0,0 +1,19 @@ +package airportsdb + +import ( + "io/fs" +) + +// Option sets an optional parameter of DB. +type Option func(*options) + +type options struct { + airportMode, countryMode fs.FileMode +} + +func defaultOpts() *options { + return &options{ + airportMode: fs.ModePerm, // 0777 + countryMode: fs.ModePerm, // 0777 + } +} diff --git a/pkg/locode/db/boltdb/calls.go b/pkg/locode/db/boltdb/calls.go new file mode 100644 index 0000000..d130097 --- /dev/null +++ b/pkg/locode/db/boltdb/calls.go @@ -0,0 +1,166 @@ +package locodebolt + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" + "go.etcd.io/bbolt" +) + +// Open opens an underlying BoltDB instance. +// +// Timeout of BoltDB opening is 3s (only for Linux or Darwin). +// +// Opens BoltDB in read-only mode if DB is read-only. +func (db *DB) Open() error { + // copy-paste from metabase: + // consider universal Open/Close for BoltDB wrappers + + err := os.MkdirAll(filepath.Dir(db.path), db.mode|0o110) + if err != nil { + return fmt.Errorf("could not create dir for BoltDB: %w", err) + } + + db.bolt, err = bbolt.Open(db.path, db.mode, db.boltOpts) + if err != nil { + return fmt.Errorf("could not open BoltDB: %w", err) + } + + return nil +} + +// Close closes an underlying BoltDB instance. +// +// Must not be called before successful Open call. +func (db *DB) Close() error { + return db.bolt.Close() +} + +func countryBucketKey(cc *locodedb.CountryCode) ([]byte, error) { + return []byte(cc.String()), nil +} + +func locationBucketKey(lc *locodedb.LocationCode) ([]byte, error) { + return []byte(lc.String()), nil +} + +type recordJSON struct { + CountryName string + LocationName string + SubDivName string + SubDivCode string + Latitude float64 + Longitude float64 + Continent string +} + +func recordValue(r locodedb.Record) ([]byte, error) { + p := r.GeoPoint() + + rj := &recordJSON{ + CountryName: r.CountryName(), + LocationName: r.LocationName(), + SubDivName: r.SubDivName(), + SubDivCode: r.SubDivCode(), + Latitude: p.Latitude(), + Longitude: p.Longitude(), + Continent: r.Continent().String(), + } + + return json.Marshal(rj) +} + +func recordFromValue(data []byte) (*locodedb.Record, error) { + rj := new(recordJSON) + + if err := json.Unmarshal(data, rj); err != nil { + return nil, err + } + + r := new(locodedb.Record) + r.SetCountryName(rj.CountryName) + r.SetLocationName(rj.LocationName) + r.SetSubDivName(rj.SubDivName) + r.SetSubDivCode(rj.SubDivCode) + r.SetGeoPoint(locodedb.NewPoint(rj.Latitude, rj.Longitude)) + + cont := locodedb.ContinentFromString(rj.Continent) + r.SetContinent(&cont) + + return r, nil +} + +// Put saves the record by key in an underlying BoltDB instance. +// +// Country code from the key is used for allocating the 1st level buckets. +// Records are stored in country buckets by the location code from the key. +// The records are stored in internal binary JSON format. +// +// Must not be called before successful Open call. +// Must not be called in read-only mode: behavior is undefined. +func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error { + return db.bolt.Batch(func(tx *bbolt.Tx) error { + countryKey, err := countryBucketKey(key.CountryCode()) + if err != nil { + return err + } + + bktCountry, err := tx.CreateBucketIfNotExists(countryKey) + if err != nil { + return fmt.Errorf("could not create country bucket: %w", err) + } + + locationKey, err := locationBucketKey(key.LocationCode()) + if err != nil { + return err + } + + cont, err := recordValue(rec) + if err != nil { + return err + } + + return bktCountry.Put(locationKey, cont) + }) +} + +var errRecordNotFound = errors.New("record not found") + +// Get reads the record by key from underlying BoltDB instance. +// +// Returns an error if no record is presented by key in DB. +// +// Must not be called before successful Open call. +func (db *DB) Get(key locodedb.Key) (rec *locodedb.Record, err error) { + err = db.bolt.View(func(tx *bbolt.Tx) error { + countryKey, err := countryBucketKey(key.CountryCode()) + if err != nil { + return err + } + + bktCountry := tx.Bucket(countryKey) + if bktCountry == nil { + return errRecordNotFound + } + + locationKey, err := locationBucketKey(key.LocationCode()) + if err != nil { + return err + } + + data := bktCountry.Get(locationKey) + if data == nil { + return errRecordNotFound + } + + rec, err = recordFromValue(data) + + return err + }) + + return +} diff --git a/pkg/locode/db/boltdb/db.go b/pkg/locode/db/boltdb/db.go new file mode 100644 index 0000000..3d09a79 --- /dev/null +++ b/pkg/locode/db/boltdb/db.go @@ -0,0 +1,73 @@ +package locodebolt + +import ( + "fmt" + "io/fs" + + "go.etcd.io/bbolt" +) + +// Prm groups the required parameters of the DB's constructor. +// +// All values must comply with the requirements imposed on them. +// Passing incorrect parameter values will result in constructor +// failure (error or panic depending on the implementation). +type Prm struct { + // Path to BoltDB file with FrostFS location database. + // + // Must not be empty. + Path string +} + +// DB is a descriptor of the FrostFS BoltDB location database. +// +// For correct operation, DB must be created +// using the constructor (New) based on the required parameters +// and optional components. +// +// After successful creation, +// DB must be opened through Open call. After successful opening, +// DB is ready to work through API (until Close call). +// +// Upon completion of work with the DB, it must be closed +// by Close method. +type DB struct { + path string + + mode fs.FileMode + + boltOpts *bbolt.Options + + bolt *bbolt.DB +} + +const invalidPrmValFmt = "invalid parameter %s (%T):%v" + +func panicOnPrmValue(n string, v any) { + panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) +} + +// New creates a new instance of the DB. +// +// Panics if at least one value of the parameters is invalid. +// +// The created DB requires calling the Open method in order +// to initialize required resources. +func New(prm Prm, opts ...Option) *DB { + switch { + case prm.Path == "": + panicOnPrmValue("Path", prm.Path) + } + + o := defaultOpts() + + for i := range opts { + opts[i](o) + } + + return &DB{ + path: prm.Path, + mode: o.mode, + boltOpts: o.boltOpts, + } +} diff --git a/pkg/locode/db/boltdb/opts.go b/pkg/locode/db/boltdb/opts.go new file mode 100644 index 0000000..db0cccd --- /dev/null +++ b/pkg/locode/db/boltdb/opts.go @@ -0,0 +1,37 @@ +package locodebolt + +import ( + "io/fs" + "os" + "time" + + "go.etcd.io/bbolt" +) + +// Option sets an optional parameter of DB. +type Option func(*options) + +type options struct { + mode fs.FileMode + + boltOpts *bbolt.Options +} + +func defaultOpts() *options { + return &options{ + mode: os.ModePerm, // 0777 + boltOpts: &bbolt.Options{ + Timeout: 3 * time.Second, + }, + } +} + +// ReadOnly enables read-only mode of the DB. +// +// Do not call DB.Put method on instances with +// this option: the behavior is undefined. +func ReadOnly() Option { + return func(o *options) { + o.boltOpts.ReadOnly = true + } +} diff --git a/pkg/locode/db/continent.go b/pkg/locode/db/continent.go new file mode 100644 index 0000000..863af7b --- /dev/null +++ b/pkg/locode/db/continent.go @@ -0,0 +1,81 @@ +package locodedb + +// Continent is an enumeration of Earth's continent. +type Continent uint8 + +const ( + // ContinentUnknown is an undefined Continent value. + ContinentUnknown = iota + + // ContinentEurope corresponds to Europe. + ContinentEurope + + // ContinentAfrica corresponds to Africa. + ContinentAfrica + + // ContinentNorthAmerica corresponds to North America. + ContinentNorthAmerica + + // ContinentSouthAmerica corresponds to South America. + ContinentSouthAmerica + + // ContinentAsia corresponds to Asia. + ContinentAsia + + // ContinentAntarctica corresponds to Antarctica. + ContinentAntarctica + + // ContinentOceania corresponds to Oceania. + ContinentOceania +) + +// Is checks if c is the same continent as c2. +func (c *Continent) Is(c2 Continent) bool { + return *c == c2 +} + +func (c Continent) String() string { + switch c { + case ContinentUnknown: + fallthrough + default: + return "Unknown" + case ContinentEurope: + return "Europe" + case ContinentAfrica: + return "Africa" + case ContinentNorthAmerica: + return "North America" + case ContinentSouthAmerica: + return "South America" + case ContinentAsia: + return "Asia" + case ContinentAntarctica: + return "Antarctica" + case ContinentOceania: + return "Oceania" + } +} + +// ContinentFromString returns Continent value +// corresponding to the passed string representation. +func ContinentFromString(str string) Continent { + switch str { + default: + return ContinentUnknown + case "Europe": + return ContinentEurope + case "Africa": + return ContinentAfrica + case "North America": + return ContinentNorthAmerica + case "South America": + return ContinentSouthAmerica + case "Asia": + return ContinentAsia + case "Antarctica": + return ContinentAntarctica + case "Oceania": + return ContinentOceania + } +} diff --git a/pkg/locode/db/continents/geojson/calls.go b/pkg/locode/db/continents/geojson/calls.go new file mode 100644 index 0000000..b5a58d1 --- /dev/null +++ b/pkg/locode/db/continents/geojson/calls.go @@ -0,0 +1,98 @@ +package continentsdb + +import ( + "fmt" + "os" + + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" + "github.com/paulmach/orb" + "github.com/paulmach/orb/geojson" + "github.com/paulmach/orb/planar" +) + +const continentProperty = "Continent" + +// PointContinent goes through all polygons and returns the continent +// in which the point is located. +// +// Returns locodedb.ContinentUnknown if no entry matches. +// +// All GeoJSON feature are parsed from file once and stored in memory. +func (db *DB) PointContinent(point *locodedb.Point) (*locodedb.Continent, error) { + var err error + + db.once.Do(func() { + err = db.init() + }) + + if err != nil { + return nil, err + } + + planarPoint := orb.Point{point.Longitude(), point.Latitude()} + + var ( + continent string + minDst float64 + ) + + for _, feature := range db.features { + if multiPolygon, ok := feature.Geometry.(orb.MultiPolygon); ok { + if planar.MultiPolygonContains(multiPolygon, planarPoint) { + continent = feature.Properties.MustString(continentProperty) + break + } + } else if polygon, ok := feature.Geometry.(orb.Polygon); ok { + if planar.PolygonContains(polygon, planarPoint) { + continent = feature.Properties.MustString(continentProperty) + break + } + } + distance := planar.DistanceFrom(feature.Geometry, planarPoint) + if minDst == 0 || minDst > distance { + minDst = distance + continent = feature.Properties.MustString(continentProperty) + } + } + + c := continentFromString(continent) + + return &c, nil +} + +func (db *DB) init() error { + data, err := os.ReadFile(db.path) + if err != nil { + return fmt.Errorf("could not read data file: %w", err) + } + + features, err := geojson.UnmarshalFeatureCollection(data) + if err != nil { + return fmt.Errorf("could not unmarshal GeoJSON feature collection: %w", err) + } + + db.features = features.Features + + return nil +} + +func continentFromString(c string) locodedb.Continent { + switch c { + default: + return locodedb.ContinentUnknown + case "Africa": + return locodedb.ContinentAfrica + case "Asia": + return locodedb.ContinentAsia + case "Europe": + return locodedb.ContinentEurope + case "North America": + return locodedb.ContinentNorthAmerica + case "South America": + return locodedb.ContinentSouthAmerica + case "Antarctica": + return locodedb.ContinentAntarctica + case "Australia", "Oceania": + return locodedb.ContinentOceania + } +} diff --git a/pkg/locode/db/continents/geojson/db.go b/pkg/locode/db/continents/geojson/db.go new file mode 100644 index 0000000..ee43bd8 --- /dev/null +++ b/pkg/locode/db/continents/geojson/db.go @@ -0,0 +1,63 @@ +package continentsdb + +import ( + "fmt" + "sync" + + "github.com/paulmach/orb/geojson" +) + +// Prm groups the required parameters of the DB's constructor. +// +// All values must comply with the requirements imposed on them. +// Passing incorrect parameter values will result in constructor +// failure (error or panic depending on the implementation). +type Prm struct { + // Path to polygons of Earth's continents in GeoJSON format. + // + // Must not be empty. + Path string +} + +// DB is a descriptor of the Earth's polygons in GeoJSON format. +// +// For correct operation, DB must be created +// using the constructor (New) based on the required parameters +// and optional components. After successful creation, +// The DB is immediately ready to work through API. +type DB struct { + path string + + once sync.Once + + features []*geojson.Feature +} + +const invalidPrmValFmt = "invalid parameter %s (%T):%v" + +func panicOnPrmValue(n string, v any) { + panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) +} + +// New creates a new instance of the DB. +// +// Panics if at least one value of the parameters is invalid. +// +// The created DB does not require additional +// initialization and is completely ready for work. +func New(prm Prm, opts ...Option) *DB { + switch { + case prm.Path == "": + panicOnPrmValue("Path", prm.Path) + } + + o := defaultOpts() + + for i := range opts { + opts[i](o) + } + + return &DB{ + path: prm.Path, + } +} diff --git a/pkg/locode/db/continents/geojson/opts.go b/pkg/locode/db/continents/geojson/opts.go new file mode 100644 index 0000000..59831fc --- /dev/null +++ b/pkg/locode/db/continents/geojson/opts.go @@ -0,0 +1,10 @@ +package continentsdb + +// Option sets an optional parameter of DB. +type Option func(*options) + +type options struct{} + +func defaultOpts() *options { + return &options{} +} diff --git a/pkg/locode/db/country.go b/pkg/locode/db/country.go new file mode 100644 index 0000000..b4f8e6c --- /dev/null +++ b/pkg/locode/db/country.go @@ -0,0 +1,32 @@ +package locodedb + +import ( + "fmt" + + locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/column" +) + +// CountryCode represents a country code for +// the storage in the FrostFS location database. +type CountryCode locodecolumn.CountryCode + +// CountryCodeFromString parses a string UN/LOCODE country code +// and returns a CountryCode. +func CountryCodeFromString(s string) (*CountryCode, error) { + cc, err := locodecolumn.CountryCodeFromString(s) + if err != nil { + return nil, fmt.Errorf("could not parse country code: %w", err) + } + + return CountryFromColumn(cc) +} + +// CountryFromColumn converts a UN/LOCODE country code to a CountryCode. +func CountryFromColumn(cc *locodecolumn.CountryCode) (*CountryCode, error) { + return (*CountryCode)(cc), nil +} + +func (c *CountryCode) String() string { + syms := (*locodecolumn.CountryCode)(c).Symbols() + return string(syms[:]) +} diff --git a/pkg/locode/db/db.go b/pkg/locode/db/db.go new file mode 100644 index 0000000..a4a09d1 --- /dev/null +++ b/pkg/locode/db/db.go @@ -0,0 +1,183 @@ +package locodedb + +import ( + "errors" + "fmt" + "runtime" + + "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" + "golang.org/x/sync/errgroup" +) + +// SourceTable is an interface of the UN/LOCODE table. +type SourceTable interface { + // Must iterate over all entries of the table + // and pass next entry to the handler. + // + // Must return handler's errors directly. + IterateAll(func(locode.Record) error) error +} + +// DB is an interface of FrostFS location database. +type DB interface { + // Must save the record by key in the database. + Put(Key, Record) error + + // Must return the record by key from the database. + Get(Key) (*Record, error) +} + +// AirportRecord represents the entry in FrostFS airport database. +type AirportRecord struct { + // Name of the country where airport is located. + CountryName string + + // Geo point where airport is located. + Point *Point +} + +// ErrAirportNotFound is returned by AirportRecord readers +// when the required airport is not found. +var ErrAirportNotFound = errors.New("airport not found") + +// AirportDB is an interface of FrostFS airport database. +type AirportDB interface { + // Must return the record by UN/LOCODE table record. + // + // Must return ErrAirportNotFound if there is no + // related airport in the database. + Get(locode.Record) (*AirportRecord, error) +} + +// ContinentsDB is an interface of FrostFS continent database. +type ContinentsDB interface { + // Must return continent of the geo point. + PointContinent(*Point) (*Continent, error) +} + +var ErrSubDivNotFound = errors.New("subdivision not found") + +var ErrCountryNotFound = errors.New("country not found") + +// NamesDB is an interface of the FrostFS location namespace. +type NamesDB interface { + // Must resolve a country code to a country name. + // + // Must return ErrCountryNotFound if there is no + // country with the provided code. + CountryName(*CountryCode) (string, error) + + // Must resolve (country code, subdivision code) to + // a subdivision name. + // + // Must return ErrSubDivNotFound if either country or + // subdivision is not presented in database. + SubDivName(*CountryCode, string) (string, error) +} + +// FillDatabase generates the FrostFS location database based on the UN/LOCODE table. +func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { + var errG errgroup.Group + + // Pick some sane default, after this the performance stopped increasing. + errG.SetLimit(runtime.NumCPU() * 4) + _ = table.IterateAll(func(tableRecord locode.Record) error { + errG.Go(func() error { + return processTableRecord(tableRecord, airports, continents, names, db) + }) + return nil + }) + return errG.Wait() +} + +func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { + if tableRecord.LOCODE.LocationCode() == "" { + return nil + } + + dbKey, err := NewKey(tableRecord.LOCODE) + if err != nil { + return err + } + + dbRecord, err := NewRecord(tableRecord) + if err != nil { + if errors.Is(err, errParseCoordinates) { + return nil + } + + return err + } + + geoPoint := dbRecord.GeoPoint() + countryName := "" + + if geoPoint == nil { + airportRecord, err := airports.Get(tableRecord) + if err != nil { + if errors.Is(err, ErrAirportNotFound) { + return nil + } + + return err + } + + geoPoint = airportRecord.Point + countryName = airportRecord.CountryName + } + + dbRecord.SetGeoPoint(geoPoint) + + if countryName == "" { + countryName, err = names.CountryName(dbKey.CountryCode()) + if err != nil { + if errors.Is(err, ErrCountryNotFound) { + return nil + } + + return err + } + } + + dbRecord.SetCountryName(countryName) + + if subDivCode := dbRecord.SubDivCode(); subDivCode != "" { + subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode) + if err != nil { + if errors.Is(err, ErrSubDivNotFound) { + return nil + } + + return err + } + + dbRecord.SetSubDivName(subDivName) + } + + continent, err := continents.PointContinent(geoPoint) + if err != nil { + return fmt.Errorf("could not calculate continent geo point: %w", err) + } else if continent.Is(ContinentUnknown) { + return nil + } + + dbRecord.SetContinent(continent) + + return db.Put(*dbKey, *dbRecord) +} + +// LocodeRecord returns the record from the FrostFS location database +// corresponding to the string representation of UN/LOCODE. +func LocodeRecord(db DB, sLocode string) (*Record, error) { + lc, err := locode.FromString(sLocode) + if err != nil { + return nil, fmt.Errorf("could not parse locode: %w", err) + } + + key, err := NewKey(*lc) + if err != nil { + return nil, err + } + + return db.Get(*key) +} diff --git a/pkg/locode/db/location.go b/pkg/locode/db/location.go new file mode 100644 index 0000000..1b08375 --- /dev/null +++ b/pkg/locode/db/location.go @@ -0,0 +1,32 @@ +package locodedb + +import ( + "fmt" + + locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/column" +) + +// LocationCode represents a location code for +// the storage in the FrostFS location database. +type LocationCode locodecolumn.LocationCode + +// LocationCodeFromString parses a string UN/LOCODE location code +// and returns a LocationCode. +func LocationCodeFromString(s string) (*LocationCode, error) { + lc, err := locodecolumn.LocationCodeFromString(s) + if err != nil { + return nil, fmt.Errorf("could not parse location code: %w", err) + } + + return LocationFromColumn(lc) +} + +// LocationFromColumn converts a UN/LOCODE country code to a LocationCode. +func LocationFromColumn(cc *locodecolumn.LocationCode) (*LocationCode, error) { + return (*LocationCode)(cc), nil +} + +func (l *LocationCode) String() string { + syms := (*locodecolumn.LocationCode)(l).Symbols() + return string(syms[:]) +} diff --git a/pkg/locode/db/point.go b/pkg/locode/db/point.go new file mode 100644 index 0000000..db47a63 --- /dev/null +++ b/pkg/locode/db/point.go @@ -0,0 +1,93 @@ +package locodedb + +import ( + "fmt" + "strconv" + + locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/column" +) + +// Point represents a 2D geographic point. +type Point struct { + lat, lng float64 +} + +// NewPoint creates, initializes and returns a new Point. +func NewPoint(lat, lng float64) *Point { + return &Point{ + lat: lat, + lng: lng, + } +} + +// Latitude returns the Point's latitude. +func (p Point) Latitude() float64 { + return p.lat +} + +// Longitude returns the Point's longitude. +func (p Point) Longitude() float64 { + return p.lng +} + +// PointFromCoordinates converts a UN/LOCODE coordinates to a Point. +func PointFromCoordinates(crd *locodecolumn.Coordinates) (*Point, error) { + if crd == nil { + return nil, nil + } + + cLat := crd.Latitude() + cLatDeg := cLat.Degrees() + cLatMnt := cLat.Minutes() + + lat, err := toDecimal(cLatDeg[:], cLatMnt[:]) + if err != nil { + return nil, fmt.Errorf("could not parse latitude: %w", err) + } + + if !cLat.Hemisphere().North() { + lat = -lat + } + + cLng := crd.Longitude() + cLngDeg := cLng.Degrees() + cLngMnt := cLng.Minutes() + + lng, err := toDecimal(cLngDeg[:], cLngMnt[:]) + if err != nil { + return nil, fmt.Errorf("could not parse longitude: %w", err) + } + + if !cLng.Hemisphere().East() { + lng = -lng + } + + return &Point{ + lat: lat, + lng: lng, + }, nil +} + +func toDecimal(intRaw, minutesRaw []byte) (float64, error) { + integer, err := strconv.ParseFloat(string(intRaw), 64) + if err != nil { + return 0, fmt.Errorf("could not parse integer part: %w", err) + } + + decimal, err := minutesToDegrees(minutesRaw) + if err != nil { + return 0, fmt.Errorf("could not parse decimal part: %w", err) + } + + return integer + decimal, nil +} + +// minutesToDegrees converts minutes to decimal part of a degree. +func minutesToDegrees(raw []byte) (float64, error) { + minutes, err := strconv.ParseFloat(string(raw), 64) + if err != nil { + return 0, err + } + + return minutes / 60, nil +} diff --git a/pkg/locode/db/point_test.go b/pkg/locode/db/point_test.go new file mode 100644 index 0000000..9531110 --- /dev/null +++ b/pkg/locode/db/point_test.go @@ -0,0 +1,51 @@ +package locodedb + +import ( + "testing" + + locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/column" + "github.com/stretchr/testify/require" +) + +func TestPointFromCoordinates(t *testing.T) { + testCases := []struct { + latGot, longGot string + latWant, longWant float64 + }{ + { + latGot: "5915N", + longGot: "01806E", + latWant: 59.25, + longWant: 18.10, + }, + { + latGot: "1000N", + longGot: "02030E", + latWant: 10.00, + longWant: 20.50, + }, + { + latGot: "0145S", + longGot: "03512W", + latWant: -01.75, + longWant: -35.20, + }, + } + + var ( + crd *locodecolumn.Coordinates + point *Point + err error + ) + + for _, test := range testCases { + crd, err = locodecolumn.CoordinatesFromString(test.latGot + " " + test.longGot) + require.NoError(t, err) + + point, err = PointFromCoordinates(crd) + require.NoError(t, err) + + require.Equal(t, test.latWant, point.Latitude()) + require.Equal(t, test.longWant, point.Longitude()) + } +} diff --git a/pkg/locode/db/record.go b/pkg/locode/db/record.go new file mode 100644 index 0000000..2699b0f --- /dev/null +++ b/pkg/locode/db/record.go @@ -0,0 +1,140 @@ +package locodedb + +import ( + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" + locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/column" +) + +// Key represents the key in FrostFS location database. +type Key struct { + cc *CountryCode + + lc *LocationCode +} + +// NewKey calculates Key from LOCODE. +func NewKey(lc locode.LOCODE) (*Key, error) { + country, err := CountryCodeFromString(lc.CountryCode()) + if err != nil { + return nil, fmt.Errorf("could not parse country: %w", err) + } + + location, err := LocationCodeFromString(lc.LocationCode()) + if err != nil { + return nil, fmt.Errorf("could not parse location: %w", err) + } + + return &Key{ + cc: country, + lc: location, + }, nil +} + +// CountryCode returns the location's country code. +func (k *Key) CountryCode() *CountryCode { + return k.cc +} + +// LocationCode returns the location code. +func (k *Key) LocationCode() *LocationCode { + return k.lc +} + +// Record represents the entry in FrostFS location database. +type Record struct { + countryName string + + locationName string + + subDivName string + + subDivCode string + + p *Point + + cont *Continent +} + +var errParseCoordinates = errors.New("invalid coordinates") + +// NewRecord calculates the Record from the UN/LOCODE table record. +func NewRecord(r locode.Record) (*Record, error) { + crd, err := locodecolumn.CoordinatesFromString(r.Coordinates) + if err != nil { + return nil, fmt.Errorf("%w: %v", errParseCoordinates, err) + } + + point, err := PointFromCoordinates(crd) + if err != nil { + return nil, fmt.Errorf("could not parse geo point: %w", err) + } + + return &Record{ + locationName: r.NameWoDiacritics, + subDivCode: r.SubDiv, + p: point, + }, nil +} + +// CountryName returns the country name. +func (r *Record) CountryName() string { + return r.countryName +} + +// SetCountryName sets the country name. +func (r *Record) SetCountryName(name string) { + r.countryName = name +} + +// LocationName returns the location name. +func (r *Record) LocationName() string { + return r.locationName +} + +// SetLocationName sets the location name. +func (r *Record) SetLocationName(name string) { + r.locationName = name +} + +// SubDivCode returns the subdivision code. +func (r *Record) SubDivCode() string { + return r.subDivCode +} + +// SetSubDivCode sets the subdivision code. +func (r *Record) SetSubDivCode(name string) { + r.subDivCode = name +} + +// SubDivName returns the subdivision name. +func (r *Record) SubDivName() string { + return r.subDivName +} + +// SetSubDivName sets the subdivision name. +func (r *Record) SetSubDivName(name string) { + r.subDivName = name +} + +// GeoPoint returns geo point of the location. +func (r *Record) GeoPoint() *Point { + return r.p +} + +// SetGeoPoint sets geo point of the location. +func (r *Record) SetGeoPoint(p *Point) { + r.p = p +} + +// Continent returns the location continent. +func (r *Record) Continent() *Continent { + return r.cont +} + +// SetContinent sets the location continent. +func (r *Record) SetContinent(c *Continent) { + r.cont = c +} diff --git a/pkg/locode/record.go b/pkg/locode/record.go new file mode 100644 index 0000000..7db746f --- /dev/null +++ b/pkg/locode/record.go @@ -0,0 +1,83 @@ +package locode + +import ( + "errors" + "fmt" + "strings" +) + +// LOCODE represents code from UN/LOCODE coding scheme. +type LOCODE [2]string + +// Record represents a single record of the UN/LOCODE table. +type Record struct { + // Change Indicator. + Ch string + + // Combination of a 2-character country code and a 3-character location code. + LOCODE LOCODE + + // Name of the locations which has been allocated a UN/LOCODE. + Name string + + // Names of the locations which have been allocated a UN/LOCODE without diacritic signs. + NameWoDiacritics string + + // ISO 1-3 character alphabetic and/or numeric code for the administrative division of the country concerned. + SubDiv string + + // 8-digit function classifier code for the location. + Function string + + // Status of the entry by a 2-character code. + Status string + + // Last date when the location was updated/entered. + Date string + + // The IATA code for the location if different from location code in column LOCODE. + IATA string + + // Geographical coordinates (latitude/longitude) of the location, if there is any. + Coordinates string + + // Some general remarks regarding the UN/LOCODE in question. + Remarks string +} + +// ErrInvalidString is the error of incorrect string format of the LOCODE. +var ErrInvalidString = errors.New("invalid string format in UN/Locode") + +// FromString parses string and returns LOCODE. +// +// If string has incorrect format, ErrInvalidString returns. +func FromString(s string) (*LOCODE, error) { + const ( + locationSeparator = " " + locodePartsNumber = 2 + ) + + words := strings.Split(s, locationSeparator) + if ln := len(words); ln != locodePartsNumber { + return nil, fmt.Errorf( + "incorrect locode: it must consist of %d codes separated with a witespase, got: %d", + locodePartsNumber, + ln, + ) + } + + l := new(LOCODE) + copy(l[:], words) + + return l, nil +} + +// CountryCode returns a string representation of country code. +func (l *LOCODE) CountryCode() string { + return l[0] +} + +// LocationCode returns a string representation of location code. +func (l *LOCODE) LocationCode() string { + return l[1] +} diff --git a/pkg/locode/table/csv/calls.go b/pkg/locode/table/csv/calls.go new file mode 100644 index 0000000..5ce0e7b --- /dev/null +++ b/pkg/locode/table/csv/calls.go @@ -0,0 +1,156 @@ +package csvlocode + +import ( + "encoding/csv" + "errors" + "io" + "os" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" +) + +var errInvalidRecord = errors.New("invalid table record") + +// IterateAll scans a table record one-by-one, parses a UN/LOCODE record +// from it and passes it to f. +// +// Returns f's errors directly. +func (t *Table) IterateAll(f func(locode.Record) error) error { + const wordsPerRecord = 12 + + return t.scanWords(t.paths, wordsPerRecord, func(words []string) error { + lc, err := locode.FromString(strings.Join(words[1:3], " ")) + if err != nil { + return err + } + + record := locode.Record{ + Ch: words[0], + LOCODE: *lc, + Name: words[3], + NameWoDiacritics: words[4], + SubDiv: words[5], + Function: words[6], + Status: words[7], + Date: words[8], + IATA: words[9], + Coordinates: words[10], + Remarks: words[11], + } + + return f(record) + }) +} + +const ( + _ = iota - 1 + + subDivCountry + subDivSubdivision + subDivName + _ // subDivLevel + + subDivFldNum +) + +type subDivKey struct { + countryCode, + subDivCode string +} + +type subDivRecord struct { + name string +} + +// SubDivName scans a table record to an in-memory table (once), +// and returns the subdivision name of the country and the subdivision codes match. +// +// Returns locodedb.ErrSubDivNotFound if no entry matches. +func (t *Table) SubDivName(countryCode *locodedb.CountryCode, code string) (string, error) { + if err := t.initSubDiv(); err != nil { + return "", err + } + + rec, ok := t.mSubDiv[subDivKey{ + countryCode: countryCode.String(), + subDivCode: code, + }] + if !ok { + return "", locodedb.ErrSubDivNotFound + } + + return rec.name, nil +} + +func (t *Table) initSubDiv() (err error) { + t.subDivOnce.Do(func() { + t.mSubDiv = make(map[subDivKey]subDivRecord) + + err = t.scanWords([]string{t.subDivPath}, subDivFldNum, func(words []string) error { + t.mSubDiv[subDivKey{ + countryCode: words[subDivCountry], + subDivCode: words[subDivSubdivision], + }] = subDivRecord{ + name: words[subDivName], + } + + return nil + }) + }) + + return +} + +var errScanInt = errors.New("interrupt scan") + +func (t *Table) scanWords(paths []string, fpr int, wordsHandler func([]string) error) error { + var ( + rdrs = make([]io.Reader, 0, len(t.paths)) + closers = make([]io.Closer, 0, len(t.paths)) + ) + + for i := range paths { + file, err := os.OpenFile(paths[i], os.O_RDONLY, t.mode) + if err != nil { + return err + } + + rdrs = append(rdrs, file) + closers = append(closers, file) + } + + defer func() { + for i := range closers { + _ = closers[i].Close() + } + }() + + r := csv.NewReader(io.MultiReader(rdrs...)) + r.ReuseRecord = true + r.FieldsPerRecord = fpr + + for { + words, err := r.Read() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + + return err + } else if len(words) != fpr { + return errInvalidRecord + } + + if err := wordsHandler(words); err != nil { + if errors.Is(err, errScanInt) { + break + } + + return err + } + } + + return nil +} diff --git a/pkg/locode/table/csv/opts.go b/pkg/locode/table/csv/opts.go new file mode 100644 index 0000000..68e4428 --- /dev/null +++ b/pkg/locode/table/csv/opts.go @@ -0,0 +1,28 @@ +package csvlocode + +import ( + "io/fs" +) + +// Option sets an optional parameter of Table. +type Option func(*options) + +type options struct { + mode fs.FileMode + + extraPaths []string +} + +func defaultOpts() *options { + return &options{ + mode: 0o700, + } +} + +// WithExtraPaths returns an option to add extra paths +// to UN/LOCODE tables in csv format. +func WithExtraPaths(ps ...string) Option { + return func(o *options) { + o.extraPaths = append(o.extraPaths, ps...) + } +} diff --git a/pkg/locode/table/csv/table.go b/pkg/locode/table/csv/table.go new file mode 100644 index 0000000..b84c2b7 --- /dev/null +++ b/pkg/locode/table/csv/table.go @@ -0,0 +1,75 @@ +package csvlocode + +import ( + "fmt" + "io/fs" + "sync" +) + +// Prm groups the required parameters of the Table's constructor. +// +// All values must comply with the requirements imposed on them. +// Passing incorrect parameter values will result in constructor +// failure (error or panic depending on the implementation). +type Prm struct { + // Path to UN/LOCODE csv table. + // + // Must not be empty. + Path string + + // Path to csv table of UN/LOCODE Subdivisions. + // + // Must not be empty. + SubDivPath string +} + +// Table is a descriptor of the UN/LOCODE table in csv format. +// +// For correct operation, Table must be created +// using the constructor (New) based on the required parameters +// and optional components. After successful creation, +// The Table is immediately ready to work through API. +type Table struct { + paths []string + + mode fs.FileMode + + subDivPath string + + subDivOnce sync.Once + + mSubDiv map[subDivKey]subDivRecord +} + +const invalidPrmValFmt = "invalid parameter %s (%T):%v" + +func panicOnPrmValue(n string, v any) { + panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) +} + +// New creates a new instance of the Table. +// +// Panics if at least one value of the parameters is invalid. +// +// The created Table does not require additional +// initialization and is completely ready for work. +func New(prm Prm, opts ...Option) *Table { + switch { + case prm.Path == "": + panicOnPrmValue("Path", prm.Path) + case prm.SubDivPath == "": + panicOnPrmValue("SubDivPath", prm.SubDivPath) + } + + o := defaultOpts() + + for i := range opts { + opts[i](o) + } + + return &Table{ + paths: append(o.extraPaths, prm.Path), + mode: o.mode, + subDivPath: prm.SubDivPath, + } +}