From 3feec01b21ff974bc187cc7ea935a750178f20ef Mon Sep 17 00:00:00 2001 From: George Bartolomey Date: Wed, 3 Jul 2024 16:39:00 +0300 Subject: [PATCH] [#6] Add automated database updater A small parser of unece.org "Trade code download page" added to Makefile to fetch last database's URL to download. Also, the forgejo action added. Action runs every month to update databases and send PR to Forgejo. Signed-off-by: George Bartolomey --- .forgejo/workflows/updatechecker.yml | 17 +++++++++++++++++ Makefile | 5 ++++- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 .forgejo/workflows/updatechecker.yml diff --git a/.forgejo/workflows/updatechecker.yml b/.forgejo/workflows/updatechecker.yml new file mode 100644 index 0000000..7eaca0f --- /dev/null +++ b/.forgejo/workflows/updatechecker.yml @@ -0,0 +1,17 @@ +on: + schedule: + - cron: "2 0 1 * *" +jobs: + checkupdates: + runs-on: docker + steps: + - uses: https://code.forgejo.org/actions/checkout@v3 + - run: make update + - run: | + git config user.name "Snegurochka" + git config user.email "snegurochka@frostfs.info" + git switch -c update-dbs + git add . + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@git.frostfs.info/TrueCloudLab/frostfs-locode-db + git commit -m "Automatic database update (UN/LOCODE version $(cat tmp/locode-version.txt))" && \ + git push origin HEAD:refs/for/master -o topic=automatic-database-update diff --git a/Makefile b/Makefile index 0415a39..1716325 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,10 @@ data/countries.dat.gz: $(DIRS) # IGNORE # See https://unece.org/trade/cefact/UNLOCODE-Download tmp/locode.csv.zip :$(DIRS) - wget -c https://service.unece.org/trade/locode/loc222csv.zip -O tmp/locode.csv.zip + DOWNLOADURL=$$(wget -O - https://unece.org/trade/cefact/UNLOCODE-Download \ + | grep -oP '(?<=href=")\S+loc\d+csv\.zip'); \ + echo "$$DOWNLOADURL" | grep -oP '(?<=loc)\d+' > tmp/locode-version.txt; \ + wget -c "$$DOWNLOADURL" -O tmp/locode.csv.zip data/unlocode-SubdivisionCodes.csv.gz: tmp/locode.csv.zip unzip -p tmp/locode.csv.zip "*SubdivisionCodes.csv" | gzip > data/unlocode-SubdivisionCodes.csv.gz