forked from TrueCloudLab/frostfs-locode-db
Compare commits
6 commits
Author | SHA1 | Date | |
---|---|---|---|
c0812d5ad6 | |||
3a73e1c89c | |||
e3d2c661e1 | |||
f99cc90185 | |||
59516714d1 | |||
abf6f2ab75 |
15 changed files with 119 additions and 39 deletions
Before Width: | Height: | Size: 5.5 KiB After Width: | Height: | Size: 5.5 KiB |
26
.forgejo/workflows/dco.yml
Normal file
26
.forgejo/workflows/dco.yml
Normal file
|
@ -0,0 +1,26 @@
|
|||
# yamllint disable rule:truthy
|
||||
|
||||
name: DCO check
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
dco:
|
||||
name: DCO
|
||||
runs-on: docker
|
||||
container:
|
||||
image: node:22-bookworm
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.22'
|
||||
|
||||
- name: Run commit format checker
|
||||
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3
|
||||
with:
|
||||
from: 'origin/${{ github.event.pull_request.base.ref }}'
|
|
@ -5,7 +5,7 @@ jobs:
|
|||
checkupdates:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v3
|
||||
- uses: actions/checkout@v3
|
||||
- run: make update
|
||||
- run: |
|
||||
git config user.name "Snegurochka"
|
||||
|
|
1
.github/CODEOWNERS
vendored
1
.github/CODEOWNERS
vendored
|
@ -1 +0,0 @@
|
|||
* @alexvanin @realloc @fyrchik @anatoly-bogatyrev
|
21
.github/workflows/dco.yml
vendored
21
.github/workflows/dco.yml
vendored
|
@ -1,21 +0,0 @@
|
|||
name: DCO check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
commits_check_job:
|
||||
runs-on: ubuntu-latest
|
||||
name: Commits Check
|
||||
steps:
|
||||
- name: Get PR Commits
|
||||
id: 'get-pr-commits'
|
||||
uses: tim-actions/get-pr-commits@master
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: DCO Check
|
||||
uses: tim-actions/dco@master
|
||||
with:
|
||||
commits: ${{ steps.get-pr-commits.outputs.commits }}
|
|
@ -1,6 +1,12 @@
|
|||
# Changelog
|
||||
Changelog for FrostFS LOCODE database
|
||||
|
||||
## 0.5.0 - 2023-09-30
|
||||
|
||||
The package now contains tool to generate database to be used
|
||||
by FrostFS inner ring nodes. The dependency on the `frostfs-cli`
|
||||
is removed from the debian package.
|
||||
|
||||
## 0.4.0 - 2023-04-24
|
||||
|
||||
Now we can build UN/LOCODE DB without Internet connection and with a
|
||||
|
|
1
CODEOWNERS
Normal file
1
CODEOWNERS
Normal file
|
@ -0,0 +1 @@
|
|||
.* @alexvanin @realloc @fyrchik @a.bogatyrev
|
|
@ -1,5 +1,5 @@
|
|||
<p align="center">
|
||||
<img src="./.github/logo.svg" width="500px" alt="NeoFS">
|
||||
<img src="./.forgejo/logo.svg" width="500px" alt="FrostFS logo">
|
||||
</p>
|
||||
<p align="center">
|
||||
UN/LOCODE database for <a href="https://frostfs.info">ForstFS</a>
|
||||
|
|
30
locode_generate_test.go
Normal file
30
locode_generate_test.go
Normal file
|
@ -0,0 +1,30 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var (
|
||||
in = flag.String(locodeGenerateInputFlag, "", "List of paths to UN/LOCODE tables (csv)")
|
||||
subdiv = flag.String(locodeGenerateSubDivFlag, "", "Path to UN/LOCODE subdivision database (csv)")
|
||||
airports = flag.String(locodeGenerateAirportsFlag, "", "Path to OpenFlights airport database (csv)")
|
||||
countries = flag.String(locodeGenerateCountriesFlag, "", "Path to OpenFlights country database (csv)")
|
||||
continents = flag.String(locodeGenerateContinentsFlag, "", "Path to continent polygons (GeoJSON)")
|
||||
out = flag.String(locodeGenerateOutputFlag, "", "Target path for generated database")
|
||||
)
|
||||
|
||||
func BenchmarkLocodeGenerate(b *testing.B) {
|
||||
locodeGenerateInPaths = append(locodeGenerateInPaths, *in)
|
||||
locodeGenerateSubDivPath = *subdiv
|
||||
locodeGenerateAirportsPath = *airports
|
||||
locodeGenerateCountriesPath = *countries
|
||||
locodeGenerateContinentsPath = *continents
|
||||
locodeGenerateOutPath = *out
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
locodeGenerateCmd.Run(locodeGenerateCmd, []string{})
|
||||
}
|
||||
}
|
|
@ -8,6 +8,7 @@ import (
|
|||
"github.com/paulmach/orb"
|
||||
"github.com/paulmach/orb/geojson"
|
||||
"github.com/paulmach/orb/planar"
|
||||
"github.com/paulmach/orb/quadtree"
|
||||
)
|
||||
|
||||
const continentProperty = "Continent"
|
||||
|
@ -36,22 +37,24 @@ func (db *DB) PointContinent(point *locodedb.Point) (*locodedb.Continent, error)
|
|||
minDst float64
|
||||
)
|
||||
|
||||
for _, feature := range db.features {
|
||||
if multiPolygon, ok := feature.Geometry.(orb.MultiPolygon); ok {
|
||||
if planar.MultiPolygonContains(multiPolygon, planarPoint) {
|
||||
pointer := db.tree.Matching(planarPoint, func(p orb.Pointer) bool {
|
||||
return planar.PolygonContains(
|
||||
p.(*geojson.Feature).Geometry.(orb.Polygon),
|
||||
planarPoint,
|
||||
)
|
||||
})
|
||||
|
||||
if pointer != nil {
|
||||
continent = pointer.(*geojson.Feature).Properties.MustString(continentProperty)
|
||||
}
|
||||
|
||||
if continent == "" {
|
||||
for _, feature := range db.features {
|
||||
distance := planar.DistanceFrom(feature.Geometry, planarPoint)
|
||||
if minDst == 0 || minDst > distance {
|
||||
minDst = distance
|
||||
continent = feature.Properties.MustString(continentProperty)
|
||||
break
|
||||
}
|
||||
} else if polygon, ok := feature.Geometry.(orb.Polygon); ok {
|
||||
if planar.PolygonContains(polygon, planarPoint) {
|
||||
continent = feature.Properties.MustString(continentProperty)
|
||||
break
|
||||
}
|
||||
}
|
||||
distance := planar.DistanceFrom(feature.Geometry, planarPoint)
|
||||
if minDst == 0 || minDst > distance {
|
||||
minDst = distance
|
||||
continent = feature.Properties.MustString(continentProperty)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -73,6 +76,39 @@ func (db *DB) init() error {
|
|||
|
||||
db.features = features.Features
|
||||
|
||||
err = db.buildQuadtree()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not build quadtree: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) buildQuadtree() error {
|
||||
db.tree = quadtree.New(orb.Bound{
|
||||
Min: orb.Point{-180, -180},
|
||||
Max: orb.Point{180, 180},
|
||||
})
|
||||
|
||||
for _, feature := range db.features {
|
||||
var multiPolygon orb.MultiPolygon
|
||||
|
||||
if polygon, ok := feature.Geometry.(orb.Polygon); ok {
|
||||
multiPolygon = append(multiPolygon, polygon)
|
||||
} else {
|
||||
multiPolygon = feature.Geometry.(orb.MultiPolygon)
|
||||
}
|
||||
|
||||
for _, polygon := range multiPolygon {
|
||||
newFeature := geojson.NewFeature(polygon)
|
||||
newFeature.Properties = feature.Properties.Clone()
|
||||
err := db.tree.Add(newFeature)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"sync"
|
||||
|
||||
"github.com/paulmach/orb/geojson"
|
||||
"github.com/paulmach/orb/quadtree"
|
||||
)
|
||||
|
||||
// Prm groups the required parameters of the DB's constructor.
|
||||
|
@ -31,6 +32,8 @@ type DB struct {
|
|||
once sync.Once
|
||||
|
||||
features []*geojson.Feature
|
||||
|
||||
tree *quadtree.Quadtree
|
||||
}
|
||||
|
||||
const invalidPrmValFmt = "invalid parameter %s (%T):%v"
|
||||
|
|
|
@ -80,7 +80,7 @@ func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB
|
|||
var errG errgroup.Group
|
||||
|
||||
// Pick some sane default, after this the performance stopped increasing.
|
||||
errG.SetLimit(runtime.NumCPU() * 4)
|
||||
errG.SetLimit(runtime.NumCPU() * 16)
|
||||
_ = table.IterateAll(func(tableRecord locode.Record) error {
|
||||
errG.Go(func() error {
|
||||
return processTableRecord(tableRecord, airports, continents, names, db)
|
||||
|
|
Loading…
Reference in a new issue