[#19] Refine CODEOWNERS settings

Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>

.forgejo/workflows/build.yml

@@ -1,38 +0,0 @@
-name: build
-on:
-  pull_request:
-  push:
-  workflow_dispatch:
-
-jobs:
-  build:
-    name: locode
-    runs-on: docker
-    container: node:22-bookworm
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.23'
-          check-latest: true
-
-      - name: Build LOCODE database
-        run: |-
-          make all
-          mkdir -p artifacts
-          mv -v frostfs-locode-db artifacts/
-          gzip --stdout --best locode_db > artifacts/locode_db.gz
-
-      - name: Publish release
-        if: >-
-          startsWith(github.ref, 'refs/tags/v') &&
-          (github.event_name == 'workflow_dispatch' || github.event_name == 'push')
-        uses: actions/release@v2
-        with:
-          direction: upload
-          release-dir: artifacts
-          token: ${{secrets.PUSH_RELEASE_TOKEN}} # user:read, repository:write

.forgejo/workflows/updatechecker.yml

@@ -1,19 +1,17 @@
 on:
   schedule:
     - cron: "2 0 1 * *"
-  workflow_dispatch:
-
 jobs:
   checkupdates:
     runs-on: docker
     steps:
-      - uses: actions/checkout@v3
+      - uses: https://code.forgejo.org/actions/checkout@v3
-      - run: make clean_data update
+      - run: make update
       - run: |
           git config user.name "Snegurochka"
           git config user.email "snegurochka@frostfs.info"
           git switch -c update-dbs
           git add .
           git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@git.frostfs.info/TrueCloudLab/frostfs-locode-db
-          git commit -sm "Automatic database update (UN/LOCODE version $(cat tmp/locode-version.txt))" && \
+          git commit -m "Automatic database update (UN/LOCODE version $(cat tmp/locode-version.txt))" && \
           git push origin HEAD:refs/for/master -o topic=automatic-database-update

CODEOWNERS

@@ -1 +1,3 @@
 .*   @alexvanin @realloc @fyrchik @a.bogatyrev
+.forgejo/.* @potyarkin
+Makefile    @potyarkin

Makefile

@@ -41,17 +41,13 @@ in/unlocode-SubdivisionCodes.csv: $(DIRS)
 in/unlocode-CodeList.csv: $(DIRS)
 	zcat data/unlocode-CodeList.csv.gz > in/unlocode-CodeList.csv
 
-in/unlocode-CodeList_overrides.csv: $(DIRS)
-	cp data/unlocode-CodeList_overrides.csv in/unlocode-CodeList_overrides.csv
-
 # Generate locode_db BoltDB file
-locode_db: in/unlocode-CodeList.csv in/unlocode-SubdivisionCodes.csv in/continents.geojson in/airports.dat in/countries.dat in/unlocode-CodeList_overrides.csv $(FROST_LOCODE)
+locode_db: in/unlocode-CodeList.csv in/unlocode-SubdivisionCodes.csv in/continents.geojson in/airports.dat in/countries.dat $(FROST_LOCODE)
 	$(FROST_LOCODE) generate \
 	--airports in/airports.dat \
 	--continents in/continents.geojson \
 	--countries in/countries.dat \
 	--in in/unlocode-CodeList.csv \
-	--overrides in/unlocode-CodeList_overrides.csv \
 	--subdiv in/unlocode-SubdivisionCodes.csv \
 	--out locode_db
 	chmod 644 locode_db
@@ -71,11 +67,8 @@ data/countries.dat.gz: $(DIRS)
 # See https://unece.org/trade/cefact/UNLOCODE-Download
 tmp/locode.csv.zip :$(DIRS)
 	DOWNLOADURL=$$(wget -O - https://unece.org/trade/cefact/UNLOCODE-Download \
-		--header='User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0' \
+		| grep -oP '(?<=href=")\S+loc\d+csv\.zip'); \
-		| grep -oP '(?<=href=")\S+loc\d+csv\.zip' \
+	echo "$$DOWNLOADURL" | grep -oP '(?<=loc)\d+' > tmp/locode-version.txt; \
-		| head -n1) && \
-	test -n "$$DOWNLOADURL" && \
-	echo "$$DOWNLOADURL" | grep -oP '(?<=loc)\d+' > tmp/locode-version.txt && \
 	wget -c "$$DOWNLOADURL" -O tmp/locode.csv.zip
 
 data/unlocode-SubdivisionCodes.csv.gz: tmp/locode.csv.zip

README.md

@@ -16,8 +16,6 @@ This repository tools generate UN/LOCODE database for FrostFS using data from
 following sources:
 - [UN/LOCODE](https://unece.org/trade/cefact/UNLOCODE-Download) database in CSV
   format, licensed under the [ODC Public Domain Dedication and Licence (PDDL)](http://opendatacommons.org/licenses/pddl/1-0/)
-- [Improved UN/LOCODE](https://raw.githubusercontent.com/cristan/improved-un-locodes/86b2e96f17289ddb62f52dcfcfac520bc1ca002b/data/code-list-improved.csv) database
-  that contains more details about LOCODEs (like coordinates) which may miss in UN/LOCODE database.
 - [OpenFlight Airports and
   Countries](https://raw.githubusercontent.com/jpatokal/openflights/master/data/)
   databases, licensed under the [GNU AGPL-3.0

locode_generate.go

@@ -16,7 +16,6 @@ type namesDB struct {
 
 const (
 	locodeGenerateInputFlag      = "in"
-	locodeGenerateOverridesFlag  = "overrides"
 	locodeGenerateSubDivFlag     = "subdiv"
 	locodeGenerateAirportsFlag   = "airports"
 	locodeGenerateCountriesFlag  = "countries"
@@ -26,7 +25,6 @@ const (
 
 var (
 	locodeGenerateInPaths        []string
-	locodeGenerateOverridesPath  string
 	locodeGenerateSubDivPath     string
 	locodeGenerateAirportsPath   string
 	locodeGenerateCountriesPath  string
@@ -37,11 +35,11 @@ var (
 		Use:   "generate",
 		Short: "Generate UN/LOCODE database for FrostFS",
 		Run: func(cmd *cobra.Command, _ []string) {
+
 			locodeDB := csvlocode.New(
 				csvlocode.Prm{
 					Path:       locodeGenerateInPaths[0],
 					SubDivPath: locodeGenerateSubDivPath,
-					OverridesPath: locodeGenerateOverridesPath,
 				},
 				csvlocode.WithExtraPaths(locodeGenerateInPaths[1:]...),
 			)
@@ -69,10 +67,8 @@ var (
 				Table: locodeDB,
 			}
 
-			res, err := locodedb.FillDatabase(locodeDB, airportDB, continentsDB, names, targetDB)
+			err = locodedb.FillDatabase(locodeDB, airportDB, continentsDB, names, targetDB)
 			ExitOnErr(cmd, "", err)
-			cmd.Printf("Records added to locode db: %d\n", res.AddedRecordCount)
-			cmd.Printf("Records ignored: %d\n", res.IgnoredRecordCount)
 		},
 	}
 )
@@ -83,8 +79,6 @@ func initUtilLocodeGenerateCmd() {
 	flags.StringSliceVar(&locodeGenerateInPaths, locodeGenerateInputFlag, nil, "List of paths to UN/LOCODE tables (csv)")
 	_ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateInputFlag)
 
-	flags.StringVar(&locodeGenerateOverridesPath, locodeGenerateOverridesFlag, "", "Path to UN/LOCODE override tables (csv)")
-
 	flags.StringVar(&locodeGenerateSubDivPath, locodeGenerateSubDivFlag, "", "Path to UN/LOCODE subdivision database (csv)")
 	_ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateSubDivFlag)
 

locode_info.go

@@ -8,7 +8,6 @@ import (
 
 const (
 	locodeInfoDBFlag   = "db"
-	locodeInfoDBFlagDesc = "Path to FrostFS UN/LOCODE database"
 	locodeInfoCodeFlag = "locode"
 )
 
@@ -48,7 +47,7 @@ var (
 func initUtilLocodeInfoCmd() {
 	flags := locodeInfoCmd.Flags()
 
-	flags.StringVar(&locodeInfoDBPath, locodeInfoDBFlag, "", locodeInfoDBFlagDesc)
+	flags.StringVar(&locodeInfoDBPath, locodeInfoDBFlag, "", "Path to FrostFS UN/LOCODE database")
 	_ = locodeInfoCmd.MarkFlagRequired(locodeInfoDBFlag)
 
 	flags.StringVar(&locodeInfoCode, locodeInfoCodeFlag, "", "UN/LOCODE")

locode_list.go

@@ -1,37 +0,0 @@
-package main
-
-import (
-	locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db"
-	locodebolt "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db/boltdb"
-	"github.com/spf13/cobra"
-)
-
-var (
-	locodeListCmd = &cobra.Command{
-		Use:   "list",
-		Short: "Print all locodes from FrostFS database",
-		Run: func(cmd *cobra.Command, _ []string) {
-			targetDB := locodebolt.New(locodebolt.Prm{
-				Path: locodeInfoDBPath,
-			}, locodebolt.ReadOnly())
-
-			err := targetDB.Open()
-			ExitOnErr(cmd, "", err)
-
-			defer targetDB.Close()
-
-			err = targetDB.IterateOverLocodes(func(locode string, geoPoint locodedb.Point) {
-				cmd.Printf("%s\t %0.2f %0.2f\n", locode, geoPoint.Latitude(), geoPoint.Longitude())
-			})
-			ExitOnErr(cmd, "", err)
-		},
-	}
-)
-
-func initUtilLocodeListCmd() {
-	flags := locodeListCmd.Flags()
-
-	flags.StringVar(&locodeInfoDBPath, locodeInfoDBFlag, "", locodeInfoDBFlagDesc)
-	_ = locodeListCmd.MarkFlagRequired(locodeInfoDBFlag)
-
-}

main.go

@@ -35,13 +35,9 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) {
 }
 
 func main() {
-	// use stdout as default output for cmd.Print()
-	rootCmd.SetOut(os.Stdout)
-
 	initUtilLocodeGenerateCmd()
 	initUtilLocodeInfoCmd()
-	initUtilLocodeListCmd()
+	rootCmd.AddCommand(locodeGenerateCmd, locodeInfoCmd)
-	rootCmd.AddCommand(locodeGenerateCmd, locodeInfoCmd, locodeListCmd)
 	err := rootCmd.Execute()
 	if err != nil {
 		ExitOnErr(rootCmd, "", err)

pkg/locode/db/boltdb/calls.go

@@ -164,23 +164,3 @@ func (db *DB) Get(key locodedb.Key) (rec *locodedb.Record, err error) {
 
 	return
 }
-
-// IterateOverLocodes iterates over all locodes.
-//
-// Returns an error if unable to unmarshal data from DB.
-//
-// Must not be called before successful Open call.
-func (db *DB) IterateOverLocodes(f func(string, locodedb.Point)) error {
-	return db.bolt.View(func(tx *bbolt.Tx) error {
-		return tx.ForEach(func(cname []byte, bktCountry *bbolt.Bucket) error {
-			return bktCountry.ForEach(func(k, v []byte) error {
-				rec, err := recordFromValue(v)
-				if err != nil {
-					return err
-				}
-				f(fmt.Sprintf("%s %s", cname, k), *rec.GeoPoint())
-				return nil
-			})
-		})
-	})
-}

pkg/locode/db/db.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"runtime"
-	"sync/atomic"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode"
 	"golang.org/x/sync/errgroup"
@@ -76,58 +75,38 @@ type NamesDB interface {
 	SubDivName(*CountryCode, string) (string, error)
 }
 
-type FillDatabaseResult struct {
-	AddedRecordCount   int
-	IgnoredRecordCount int
-}
-
 // FillDatabase generates the FrostFS location database based on the UN/LOCODE table.
-func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) (FillDatabaseResult, error) {
+func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
 	var errG errgroup.Group
 
-	var added, ignored atomic.Int32
-
 	// Pick some sane default, after this the performance stopped increasing.
 	errG.SetLimit(runtime.NumCPU() * 16)
 	_ = table.IterateAll(func(tableRecord locode.Record) error {
 		errG.Go(func() error {
-			wasAdded, err := processTableRecord(tableRecord, airports, continents, names, db)
+			return processTableRecord(tableRecord, airports, continents, names, db)
-			if err != nil {
-				return err
-			}
-			if wasAdded {
-				added.Add(1)
-			} else {
-				ignored.Add(1)
-			}
-			return nil
 		})
 		return nil
 	})
-
+	return errG.Wait()
-	return FillDatabaseResult{
-		AddedRecordCount:   int(added.Load()),
-		IgnoredRecordCount: int(ignored.Load()),
-	}, errG.Wait()
 }
 
-func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) (bool, error) {
+func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error {
 	if tableRecord.LOCODE.LocationCode() == "" {
-		return false, nil
+		return nil
 	}
 
 	dbKey, err := NewKey(tableRecord.LOCODE)
 	if err != nil {
-		return false, err
+		return err
 	}
 
 	dbRecord, err := NewRecord(tableRecord)
 	if err != nil {
 		if errors.Is(err, errParseCoordinates) {
-			return false, nil
+			return nil
 		}
 
-		return false, err
+		return err
 	}
 
 	geoPoint := dbRecord.GeoPoint()
@@ -137,10 +116,10 @@ func processTableRecord(tableRecord locode.Record, airports AirportDB, continent
 		airportRecord, err := airports.Get(tableRecord)
 		if err != nil {
 			if errors.Is(err, ErrAirportNotFound) {
-				return false, nil
+				return nil
 			}
 
-			return false, err
+			return err
 		}
 
 		geoPoint = airportRecord.Point
@@ -153,10 +132,10 @@ func processTableRecord(tableRecord locode.Record, airports AirportDB, continent
 		countryName, err = names.CountryName(dbKey.CountryCode())
 		if err != nil {
 			if errors.Is(err, ErrCountryNotFound) {
-				return false, nil
+				return nil
 			}
 
-			return false, err
+			return err
 		}
 	}
 
@@ -166,10 +145,10 @@ func processTableRecord(tableRecord locode.Record, airports AirportDB, continent
 		subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode)
 		if err != nil {
 			if errors.Is(err, ErrSubDivNotFound) {
-				return false, nil
+				return nil
 			}
 
-			return false, err
+			return err
 		}
 
 		dbRecord.SetSubDivName(subDivName)
@@ -177,14 +156,14 @@ func processTableRecord(tableRecord locode.Record, airports AirportDB, continent
 
 	continent, err := continents.PointContinent(geoPoint)
 	if err != nil {
-		return false, fmt.Errorf("could not calculate continent geo point: %w", err)
+		return fmt.Errorf("could not calculate continent geo point: %w", err)
 	} else if continent.Is(ContinentUnknown) {
-		return false, nil
+		return nil
 	}
 
 	dbRecord.SetContinent(continent)
 
-	return true, db.Put(*dbKey, *dbRecord)
+	return db.Put(*dbKey, *dbRecord)
 }
 
 // LocodeRecord returns the record from the FrostFS location database

pkg/locode/table/csv/calls.go

@@ -3,7 +3,6 @@ package csvlocode
 import (
 	"encoding/csv"
 	"errors"
-	"fmt"
 	"io"
 	"os"
 	"strings"
@@ -41,10 +40,6 @@ func (t *Table) IterateAll(f func(locode.Record) error) error {
 			Remarks:          words[11],
 		}
 
-		if err := t.Override(&record); err != nil {
-			return fmt.Errorf("override: %w", err)
-		}
-
 		return f(record)
 	})
 }
@@ -89,14 +84,6 @@ func (t *Table) SubDivName(countryCode *locodedb.CountryCode, code string) (stri
 	return rec.name, nil
 }
 
-func (t *Table) Override(record *locode.Record) error {
-	override, found := t.overrides[record.LOCODE]
-	if found {
-		*record = override
-	}
-	return nil
-}
-
 func (t *Table) initSubDiv() (err error) {
 	t.subDivOnce.Do(func() {
 		t.mSubDiv = make(map[subDivKey]subDivRecord)
@@ -116,40 +103,6 @@ func (t *Table) initSubDiv() (err error) {
 	return
 }
 
-func (t *Table) initOverrides(overridesPath string) error {
-	const wordsPerRecord = 12
-
-	t.overrides = make(map[locode.LOCODE]locode.Record)
-
-	if overridesPath == "" {
-		return nil
-	}
-
-	return t.scanWords([]string{overridesPath}, wordsPerRecord, func(words []string) error {
-		lc, err := locode.FromString(strings.Join(words[1:3], " "))
-		if err != nil {
-			return err
-		}
-
-		record := locode.Record{
-			Ch:               words[0],
-			LOCODE:           *lc,
-			Name:             words[3],
-			NameWoDiacritics: words[4],
-			SubDiv:           words[5],
-			Function:         words[6],
-			Status:           words[7],
-			Date:             words[8],
-			IATA:             words[9],
-			Coordinates:      words[10],
-			Remarks:          words[11],
-		}
-
-		t.overrides[record.LOCODE] = record
-		return nil
-	})
-}
-
 var errScanInt = errors.New("interrupt scan")
 
 func (t *Table) scanWords(paths []string, fpr int, wordsHandler func([]string) error) error {

pkg/locode/table/csv/table.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"io/fs"
 	"sync"
-
-	"git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode"
 )
 
 // Prm groups the required parameters of the Table's constructor.
@@ -23,11 +21,6 @@ type Prm struct {
 	//
 	// Must not be empty.
 	SubDivPath string
-
-	// Path to a csv table with UN/LOCODE overrides.
-	//
-	// Optional.
-	OverridesPath string
 }
 
 // Table is a descriptor of the UN/LOCODE table in csv format.
@@ -46,8 +39,6 @@ type Table struct {
 	subDivOnce sync.Once
 
 	mSubDiv map[subDivKey]subDivRecord
-
-	overrides map[locode.LOCODE]locode.Record
 }
 
 const invalidPrmValFmt = "invalid parameter %s (%T):%v"
@@ -76,15 +67,9 @@ func New(prm Prm, opts ...Option) *Table {
 		opts[i](o)
 	}
 
-	t := &Table{
+	return &Table{
 		paths:      append(o.extraPaths, prm.Path),
 		mode:       o.mode,
 		subDivPath: prm.SubDivPath,
 	}
-
-	if err := t.initOverrides(prm.OverridesPath); err != nil {
-		panic(fmt.Errorf("init overrides: %w", err))
-	}
-
-	return t
 }