From 42a6c6904bcebae273098c6574f99fad306e079e Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Thu, 13 Mar 2025 12:47:21 +0300 Subject: [PATCH] [#26] ci: Revive automatic database updates Database updates were broken for a while (and noone noticed because CI was not even switched on). This commit: - Adds some greyhat web scraping (fake User-Agent to avoid HTTP 403) - Surfaces errors in intermediate recipe steps (`cmd && cmd` instead of `cmd; cmd`) - Works around other possible CI errors in automatic PRs (--signoff) Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/updatechecker.yml | 4 ++-- Makefile | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.forgejo/workflows/updatechecker.yml b/.forgejo/workflows/updatechecker.yml index 73d216e..4ddb563 100644 --- a/.forgejo/workflows/updatechecker.yml +++ b/.forgejo/workflows/updatechecker.yml @@ -6,12 +6,12 @@ jobs: runs-on: docker steps: - uses: actions/checkout@v3 - - run: make update + - run: make clean_data update - run: | git config user.name "Snegurochka" git config user.email "snegurochka@frostfs.info" git switch -c update-dbs git add . git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@git.frostfs.info/TrueCloudLab/frostfs-locode-db - git commit -m "Automatic database update (UN/LOCODE version $(cat tmp/locode-version.txt))" && \ + git commit -sm "Automatic database update (UN/LOCODE version $(cat tmp/locode-version.txt))" && \ git push origin HEAD:refs/for/master -o topic=automatic-database-update diff --git a/Makefile b/Makefile index 27428ad..925201d 100755 --- a/Makefile +++ b/Makefile @@ -71,8 +71,11 @@ data/countries.dat.gz: $(DIRS) # See https://unece.org/trade/cefact/UNLOCODE-Download tmp/locode.csv.zip :$(DIRS) DOWNLOADURL=$$(wget -O - https://unece.org/trade/cefact/UNLOCODE-Download \ - | grep -oP '(?<=href=")\S+loc\d+csv\.zip'); \ - echo "$$DOWNLOADURL" | grep -oP '(?<=loc)\d+' > tmp/locode-version.txt; \ + --header='User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0' \ + | grep -oP '(?<=href=")\S+loc\d+csv\.zip' \ + | head -n1) && \ + test -n "$$DOWNLOADURL" && \ + echo "$$DOWNLOADURL" | grep -oP '(?<=loc)\d+' > tmp/locode-version.txt && \ wget -c "$$DOWNLOADURL" -O tmp/locode.csv.zip data/unlocode-SubdivisionCodes.csv.gz: tmp/locode.csv.zip