From 7578fd46e594188666120dbe20a94fd59bb2ea7f Mon Sep 17 00:00:00 2001 From: nesterfifa Date: Tue, 28 Nov 2023 12:33:42 +0100 Subject: [PATCH 1/6] [#782] pkg: use quadtree to find continent Signed-off-by: nesterfifa --- pkg/services/control/ir/service.pb.go | Bin 33564 -> 33565 bytes pkg/services/control/ir/service_grpc.pb.go | Bin 7044 -> 7045 bytes pkg/services/control/ir/types.pb.go | Bin 7799 -> 7800 bytes pkg/services/control/service.pb.go | Bin 146242 -> 146243 bytes pkg/services/control/service_grpc.pb.go | Bin 23641 -> 23642 bytes pkg/services/control/types.pb.go | Bin 29301 -> 29302 bytes pkg/services/tree/service.pb.go | Bin 124265 -> 124266 bytes pkg/services/tree/service_grpc.pb.go | Bin 19139 -> 19140 bytes pkg/services/tree/types.pb.go | Bin 9911 -> 9912 bytes .../locode/db/continents/geojson/calls.go | 64 ++++++++++++++---- pkg/util/locode/db/continents/geojson/db.go | 3 + 11 files changed, 53 insertions(+), 14 deletions(-) diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index bec74a3beaecfdcc8792ed3dc368d640e5f72eac..a138bbd84b874e4e3ccd908c9c9e1d58deb038db 100644 GIT binary patch delta 20 ccmbQ!#x%E$X~O==mW=Wf_f~I?U_85UIp9aYyy$U`C}v0QK<_rT_o{ delta 19 bcmezNgz@VW#tFM7-maXuQ(+mGQ__#tkftY+Rh_MFq)|0~jSXD>5$f1ORSJ2kHO- delta 24 gcmX>ymGSUY#tkftteojZ1<8{G86`F=F)s210BS}D;Q#;t diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go index b4d6981ef92d8434e0d3ffa6370d8cb860db9ce4..873780ec69a03e92d445f057650c616ec60f7200 100644 GIT binary patch delta 18 acmdn)yTf&dB%@)KWHZ*E~URs{f7AqXM> delta 18 acmdntyWMxf>&ZNfl@nhnY;I*VRs{f4V+YCr diff --git a/pkg/util/locode/db/continents/geojson/calls.go b/pkg/util/locode/db/continents/geojson/calls.go index 34467d5a2..5cd0820a7 100644 --- a/pkg/util/locode/db/continents/geojson/calls.go +++ b/pkg/util/locode/db/continents/geojson/calls.go @@ -4,6 +4,8 @@ import ( "fmt" "os" + "github.com/paulmach/orb/quadtree" + locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" @@ -36,22 +38,24 @@ func (db *DB) PointContinent(point *locodedb.Point) (*locodedb.Continent, error) minDst float64 ) - for _, feature := range db.features { - if multiPolygon, ok := feature.Geometry.(orb.MultiPolygon); ok { - if planar.MultiPolygonContains(multiPolygon, planarPoint) { + pointer := db.tree.Matching(planarPoint, func(p orb.Pointer) bool { + return planar.PolygonContains( + p.(*geojson.Feature).Geometry.(orb.Polygon), + planarPoint, + ) + }) + + if pointer != nil { + continent = pointer.(*geojson.Feature).Properties.MustString(continentProperty) + } + + if continent == "" { + for _, feature := range db.features { + distance := planar.DistanceFrom(feature.Geometry, planarPoint) + if minDst == 0 || minDst > distance { + minDst = distance continent = feature.Properties.MustString(continentProperty) - break } - } else if polygon, ok := feature.Geometry.(orb.Polygon); ok { - if planar.PolygonContains(polygon, planarPoint) { - continent = feature.Properties.MustString(continentProperty) - break - } - } - distance := planar.DistanceFrom(feature.Geometry, planarPoint) - if minDst == 0 || minDst > distance { - minDst = distance - continent = feature.Properties.MustString(continentProperty) } } @@ -73,6 +77,38 @@ func (db *DB) init() error { db.features = features.Features + err = db.buildQuadtree() + if err != nil { + return fmt.Errorf("could not build quadtree: %w", err) + } + + return nil +} + +func (db *DB) buildQuadtree() error { + db.tree = quadtree.New(orb.Bound{ + Min: orb.Point{-180, -180}, + Max: orb.Point{180, 180}, + }) + + for _, feature := range db.features { + if multiPolygon, ok := feature.Geometry.(orb.MultiPolygon); ok { + for _, polygon := range multiPolygon { + newFeature := geojson.NewFeature(polygon) + newFeature.Properties = feature.Properties.Clone() + err := db.tree.Add(newFeature) + if err != nil { + return err + } + } + } else if _, ok := feature.Geometry.(orb.Polygon); ok { + err := db.tree.Add(feature) + if err != nil { + return err + } + } + } + return nil } diff --git a/pkg/util/locode/db/continents/geojson/db.go b/pkg/util/locode/db/continents/geojson/db.go index ee43bd810..244da024f 100644 --- a/pkg/util/locode/db/continents/geojson/db.go +++ b/pkg/util/locode/db/continents/geojson/db.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/paulmach/orb/geojson" + "github.com/paulmach/orb/quadtree" ) // Prm groups the required parameters of the DB's constructor. @@ -31,6 +32,8 @@ type DB struct { once sync.Once features []*geojson.Feature + + tree *quadtree.Quadtree } const invalidPrmValFmt = "invalid parameter %s (%T):%v" -- 2.45.2 From cc27dbcb789d3e78c63028c952b607aa20ddc3f5 Mon Sep 17 00:00:00 2001 From: nesterfifa Date: Mon, 4 Dec 2023 17:29:22 +0100 Subject: [PATCH 2/6] [#782] locode: Add benchmark Signed-off-by: nesterfifa --- .../modules/util/locode_generate_test.go | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 cmd/frostfs-cli/modules/util/locode_generate_test.go diff --git a/cmd/frostfs-cli/modules/util/locode_generate_test.go b/cmd/frostfs-cli/modules/util/locode_generate_test.go new file mode 100644 index 000000000..722b303d4 --- /dev/null +++ b/cmd/frostfs-cli/modules/util/locode_generate_test.go @@ -0,0 +1,30 @@ +package util + +import ( + "flag" + "testing" +) + +var ( + in = flag.String(locodeGenerateInputFlag, "", "List of paths to UN/LOCODE tables (csv)") + subdiv = flag.String(locodeGenerateSubDivFlag, "", "Path to UN/LOCODE subdivision database (csv)") + airports = flag.String(locodeGenerateAirportsFlag, "", "Path to OpenFlights airport database (csv)") + countries = flag.String(locodeGenerateCountriesFlag, "", "Path to OpenFlights country database (csv)") + continents = flag.String(locodeGenerateContinentsFlag, "", "Path to continent polygons (GeoJSON)") + out = flag.String(locodeGenerateOutputFlag, "", "Target path for generated database") +) + +func BenchmarkLocodeGenerate(b *testing.B) { + locodeGenerateInPaths = append(locodeGenerateInPaths, *in) + locodeGenerateSubDivPath = *subdiv + locodeGenerateAirportsPath = *airports + locodeGenerateCountriesPath = *countries + locodeGenerateContinentsPath = *continents + locodeGenerateOutPath = *out + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + locodeGenerateCmd.Run(locodeGenerateCmd, []string{}) + } +} -- 2.45.2 From 03d32fa950234f41c93dda3b23bf3105d1fd3135 Mon Sep 17 00:00:00 2001 From: nesterfifa Date: Mon, 4 Dec 2023 17:29:56 +0100 Subject: [PATCH 3/6] [#782] locode: Increase number of CPUs Signed-off-by: nesterfifa --- pkg/util/locode/db/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/locode/db/db.go b/pkg/util/locode/db/db.go index 8c71ea794..aea5caa59 100644 --- a/pkg/util/locode/db/db.go +++ b/pkg/util/locode/db/db.go @@ -80,7 +80,7 @@ func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB var errG errgroup.Group // Pick some sane default, after this the performance stopped increasing. - errG.SetLimit(runtime.NumCPU() * 4) + errG.SetLimit(runtime.NumCPU() * 16) _ = table.IterateAll(func(tableRecord locode.Record) error { errG.Go(func() error { return processTableRecord(tableRecord, airports, continents, names, db) -- 2.45.2 From 91fe0d353140e24e84cf73ac69e52363d373dac9 Mon Sep 17 00:00:00 2001 From: nesterfifa Date: Mon, 4 Dec 2023 17:41:16 +0100 Subject: [PATCH 4/6] [#782] node: Discard changes in gen files Signed-off-by: nesterfifa --- pkg/services/control/ir/service.pb.go | Bin 33565 -> 33564 bytes pkg/services/control/ir/service_grpc.pb.go | Bin 7045 -> 7044 bytes pkg/services/control/ir/types.pb.go | Bin 7800 -> 7799 bytes pkg/services/control/service.pb.go | Bin 146243 -> 146242 bytes pkg/services/control/service_grpc.pb.go | Bin 23642 -> 23641 bytes pkg/services/control/types.pb.go | Bin 29302 -> 29301 bytes pkg/services/tree/service.pb.go | Bin 124266 -> 124265 bytes pkg/services/tree/service_grpc.pb.go | Bin 19140 -> 19139 bytes pkg/services/tree/types.pb.go | Bin 9912 -> 9911 bytes 9 files changed, 0 insertions(+), 0 deletions(-) diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index a138bbd84b874e4e3ccd908c9c9e1d58deb038db..bec74a3beaecfdcc8792ed3dc368d640e5f72eac 100644 GIT binary patch delta 19 bcmbQ+#x$pmX~O=A4=N|_Q`j8Ic%%^kTjvQx delta 20 ccmbQ!#x%E$X~O==mW=Wf_f~I?U_85UIp9aYyy$U`C}v0QK<_rT_o{ diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index 74adec45ab8d3bfb4ca054032440dde585145b84..63f3e714a2578d648309f46e53a12d3eda9f61f0 100644 GIT binary patch delta 24 gcmaELiv8s&_6d(BGc#6Be5lY|!??YMk@4MQ0GOEzkN^Mx delta 24 gcmaEPiv86o_6d(B$1uuId{Et7&A7dqk@4MQ0GzQ4=Kufz diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index f05ab11ed23f6cdbb2f162c1d5d81957218372e9..2c0828951aea66870e38f68316b12ef987686967 100644 GIT binary patch delta 24 gcmX>ymGSUY#tkftteojZ1<8{G86`F=F)s210BS}D;Q#;t delta 25 hcmX>+mGQ__#tkftY+Rh_MFq)|0~jSXD>5$f1ORSJ2kHO- diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go index 873780ec69a03e92d445f057650c616ec60f7200..b4d6981ef92d8434e0d3ffa6370d8cb860db9ce4 100644 GIT binary patch delta 18 acmdntyWMxf>&ZNfl@nhnY;I*VRs{f4V+YCr delta 18 acmdn)yTf&dB%@)KWHZ*E~URs{f7AqXM> -- 2.45.2 From f31b4c1a7b129ef092915c1b6ede4d2ce0ded565 Mon Sep 17 00:00:00 2001 From: nesterfifa Date: Tue, 12 Dec 2023 11:20:56 +0100 Subject: [PATCH 5/6] [#782] locode: Simplify build quadtree Signed-off-by: nesterfifa --- .../locode/db/continents/geojson/calls.go | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/util/locode/db/continents/geojson/calls.go b/pkg/util/locode/db/continents/geojson/calls.go index 5cd0820a7..6519b1aa8 100644 --- a/pkg/util/locode/db/continents/geojson/calls.go +++ b/pkg/util/locode/db/continents/geojson/calls.go @@ -92,17 +92,18 @@ func (db *DB) buildQuadtree() error { }) for _, feature := range db.features { - if multiPolygon, ok := feature.Geometry.(orb.MultiPolygon); ok { - for _, polygon := range multiPolygon { - newFeature := geojson.NewFeature(polygon) - newFeature.Properties = feature.Properties.Clone() - err := db.tree.Add(newFeature) - if err != nil { - return err - } - } - } else if _, ok := feature.Geometry.(orb.Polygon); ok { - err := db.tree.Add(feature) + multiPolygon := make(orb.MultiPolygon, 0) + + if polygon, ok := feature.Geometry.(orb.Polygon); ok { + multiPolygon = append(multiPolygon, polygon) + } else { + multiPolygon = feature.Geometry.(orb.MultiPolygon) + } + + for _, polygon := range multiPolygon { + newFeature := geojson.NewFeature(polygon) + newFeature.Properties = feature.Properties.Clone() + err := db.tree.Add(newFeature) if err != nil { return err } -- 2.45.2 From 6999993fd20223573f56e32ad72acff1d2d5730d Mon Sep 17 00:00:00 2001 From: nesterfifa Date: Tue, 12 Dec 2023 12:34:04 +0100 Subject: [PATCH 6/6] [#782] locode: Fix imports, var init Signed-off-by: nesterfifa --- pkg/util/locode/db/continents/geojson/calls.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/util/locode/db/continents/geojson/calls.go b/pkg/util/locode/db/continents/geojson/calls.go index 6519b1aa8..bf85d3f87 100644 --- a/pkg/util/locode/db/continents/geojson/calls.go +++ b/pkg/util/locode/db/continents/geojson/calls.go @@ -4,12 +4,11 @@ import ( "fmt" "os" - "github.com/paulmach/orb/quadtree" - locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" "github.com/paulmach/orb/planar" + "github.com/paulmach/orb/quadtree" ) const continentProperty = "Continent" @@ -92,7 +91,7 @@ func (db *DB) buildQuadtree() error { }) for _, feature := range db.features { - multiPolygon := make(orb.MultiPolygon, 0) + var multiPolygon orb.MultiPolygon if polygon, ok := feature.Geometry.(orb.Polygon); ok { multiPolygon = append(multiPolygon, polygon) -- 2.45.2