[#25] ci: Publish release artifacts automatically

Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>

.forgejo/workflows/build.yml

@@ -0,0 +1,38 @@
+name: build
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: locode
+    runs-on: docker
+    container: node:22-bookworm
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.23'
+          check-latest: true
+
+      - name: Build LOCODE database
+        run: |-
+          make all
+          mkdir -p artifacts
+          mv -v frostfs-locode-db artifacts/
+          gzip --stdout --best locode_db > artifacts/locode_db.gz
+
+      - name: Publish release
+        if: >-
+          startsWith(github.ref, 'refs/tags/v') &&
+          (github.event_name == 'workflow_dispatch' || github.event_name == 'push')
+        uses: actions/release@v2
+        with:
+          direction: upload
+          release-dir: artifacts
+          token: ${{secrets.PUSH_RELEASE_TOKEN}} # user:read, repository:write

.forgejo/workflows/updatechecker.yml

@@ -1,17 +1,19 @@
 on:
   schedule:
     - cron: "2 0 1 * *"
+  workflow_dispatch:
+
 jobs:
   checkupdates:
     runs-on: docker
     steps:
       - uses: actions/checkout@v3
-      - run: make update
+      - run: make clean_data update
       - run: |
           git config user.name "Snegurochka"
           git config user.email "snegurochka@frostfs.info"
           git switch -c update-dbs
           git add .
           git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@git.frostfs.info/TrueCloudLab/frostfs-locode-db
-          git commit -m "Automatic database update (UN/LOCODE version $(cat tmp/locode-version.txt))" && \
+          git commit -sm "Automatic database update (UN/LOCODE version $(cat tmp/locode-version.txt))" && \
           git push origin HEAD:refs/for/master -o topic=automatic-database-update

Makefile

@@ -71,8 +71,11 @@ data/countries.dat.gz: $(DIRS)
 # See https://unece.org/trade/cefact/UNLOCODE-Download
 tmp/locode.csv.zip :$(DIRS)
 	DOWNLOADURL=$$(wget -O - https://unece.org/trade/cefact/UNLOCODE-Download \
-		| grep -oP '(?<=href=")\S+loc\d+csv\.zip'); \
-	echo "$$DOWNLOADURL" | grep -oP '(?<=loc)\d+' > tmp/locode-version.txt; \
+		--header='User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0' \
+		| grep -oP '(?<=href=")\S+loc\d+csv\.zip' \
+		| head -n1) && \
+	test -n "$$DOWNLOADURL" && \
+	echo "$$DOWNLOADURL" | grep -oP '(?<=loc)\d+' > tmp/locode-version.txt && \
 	wget -c "$$DOWNLOADURL" -O tmp/locode.csv.zip
 
 data/unlocode-SubdivisionCodes.csv.gz: tmp/locode.csv.zip

locode_info.go

@@ -7,8 +7,9 @@ import (
 )
 
 const (
-	locodeInfoDBFlag   = "db"
-	locodeInfoCodeFlag = "locode"
+	locodeInfoDBFlag     = "db"
+	locodeInfoDBFlagDesc = "Path to FrostFS UN/LOCODE database"
+	locodeInfoCodeFlag   = "locode"
 )
 
 var (
@@ -47,7 +48,7 @@ var (
 func initUtilLocodeInfoCmd() {
 	flags := locodeInfoCmd.Flags()
 
-	flags.StringVar(&locodeInfoDBPath, locodeInfoDBFlag, "", "Path to FrostFS UN/LOCODE database")
+	flags.StringVar(&locodeInfoDBPath, locodeInfoDBFlag, "", locodeInfoDBFlagDesc)
 	_ = locodeInfoCmd.MarkFlagRequired(locodeInfoDBFlag)
 
 	flags.StringVar(&locodeInfoCode, locodeInfoCodeFlag, "", "UN/LOCODE")

locode_list.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db"
+	locodebolt "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db/boltdb"
+	"github.com/spf13/cobra"
+)
+
+var (
+	locodeListCmd = &cobra.Command{
+		Use:   "list",
+		Short: "Print all locodes from FrostFS database",
+		Run: func(cmd *cobra.Command, _ []string) {
+			targetDB := locodebolt.New(locodebolt.Prm{
+				Path: locodeInfoDBPath,
+			}, locodebolt.ReadOnly())
+
+			err := targetDB.Open()
+			ExitOnErr(cmd, "", err)
+
+			defer targetDB.Close()
+
+			err = targetDB.IterateOverLocodes(func(locode string, geoPoint locodedb.Point) {
+				cmd.Printf("%s\t %0.2f %0.2f\n", locode, geoPoint.Latitude(), geoPoint.Longitude())
+			})
+			ExitOnErr(cmd, "", err)
+		},
+	}
+)
+
+func initUtilLocodeListCmd() {
+	flags := locodeListCmd.Flags()
+
+	flags.StringVar(&locodeInfoDBPath, locodeInfoDBFlag, "", locodeInfoDBFlagDesc)
+	_ = locodeListCmd.MarkFlagRequired(locodeInfoDBFlag)
+
+}

main.go

@@ -35,9 +35,13 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) {
 }
 
 func main() {
+	// use stdout as default output for cmd.Print()
+	rootCmd.SetOut(os.Stdout)
+
 	initUtilLocodeGenerateCmd()
 	initUtilLocodeInfoCmd()
-	rootCmd.AddCommand(locodeGenerateCmd, locodeInfoCmd)
+	initUtilLocodeListCmd()
+	rootCmd.AddCommand(locodeGenerateCmd, locodeInfoCmd, locodeListCmd)
 	err := rootCmd.Execute()
 	if err != nil {
 		ExitOnErr(rootCmd, "", err)

pkg/locode/db/boltdb/calls.go

@@ -164,3 +164,23 @@ func (db *DB) Get(key locodedb.Key) (rec *locodedb.Record, err error) {
 
 	return
 }
+
+// IterateOverLocodes iterates over all locodes.
+//
+// Returns an error if unable to unmarshal data from DB.
+//
+// Must not be called before successful Open call.
+func (db *DB) IterateOverLocodes(f func(string, locodedb.Point)) error {
+	return db.bolt.View(func(tx *bbolt.Tx) error {
+		return tx.ForEach(func(cname []byte, bktCountry *bbolt.Bucket) error {
+			return bktCountry.ForEach(func(k, v []byte) error {
+				rec, err := recordFromValue(v)
+				if err != nil {
+					return err
+				}
+				f(fmt.Sprintf("%s %s", cname, k), *rec.GeoPoint())
+				return nil
+			})
+		})
+	})
+}