[#5] Add local data sources

To simplify build process, we add data sources in a compressed form
directly in to our git repository.

Signed-off-by: Stanislav Bogatyrev <s.bogatyrev@yadro.com>

Makefile

@@ -3,10 +3,11 @@
 VERSION ?= "$(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")"
 FROST ?= frostfs-cli
 
-.PHONY: all clean version help unlocode debpackage
+.PHONY: all clean version help update debpackage
 
 DIRS = in tmp
 
+# IGNORE
 space := $(subst ,, )
 
 # .deb package versioning
@@ -15,6 +16,7 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
 	sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
 	sed "s/-/~/")-${OS_RELEASE}
 
+# Generate locode_db BoltDB file
 all: $(DIRS) locode_db
 
 $(DIRS):
@@ -22,28 +24,56 @@ $(DIRS):
 	@mkdir -p $@
 
 in/airports.dat: $(DIRS)
-	wget -c https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat -O in/airports.dat
+	zcat data/airports.dat.gz > in/airports.dat
 
 in/countries.dat: $(DIRS)
-	wget -c https://raw.githubusercontent.com/jpatokal/openflights/master/data/countries.dat -O in/countries.dat
+	zcat data/countries.dat.gz > in/countries.dat
 
 in/continents.geojson: $(DIRS)
-	zcat continents.geojson.gz > in/continents.geojson
+	zcat data/continents.geojson.gz > in/continents.geojson
 
-unlocode :$(DIRS)
+in/unlocode-SubdivisionCodes.csv: $(DIRS)
-	wget -c https://service.unece.org/trade/locode/loc222csv.zip -O tmp/loc222csv.zip
+	zcat data/unlocode-SubdivisionCodes.csv.gz > in/unlocode-SubdivisionCodes.csv
-	unzip -u tmp/loc222csv.zip -d in/
 
-locode_db: unlocode in/continents.geojson in/airports.dat in/countries.dat
+in/unlocode-CodeList.csv: $(DIRS)
+	zcat data/unlocode-CodeList.csv.gz > in/unlocode-CodeList.csv
+
+# Generate locode_db BoltDB file
+locode_db: in/unlocode-CodeList.csv in/unlocode-SubdivisionCodes.csv in/continents.geojson in/airports.dat in/countries.dat
 	$(FROST) util locode generate \
 	--airports in/airports.dat \
 	--continents in/continents.geojson \
 	--countries in/countries.dat \
-	--in in/2022-2\ UNLOCODE\ CodeListPart1.csv,in/2022-2\ UNLOCODE\ CodeListPart2.csv,in/2022-2\ UNLOCODE\ CodeListPart3.csv \
+	--in in/unlocode-CodeList.csv \
-	--subdiv in/2022-2\ SubdivisionCodes.csv \
+	--subdiv in/unlocode-SubdivisionCodes.csv \
 	--out locode_db
 	chmod 644 locode_db
 
+# IGNORE
+# Openflights doesn't have tags or releases, so update would always take latest master version.
+# See https://github.com/jpatokal/openflights
+data/airports.dat.gz: $(DIRS)
+	wget -c https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat -O tmp/airports.dat && \
+  gzip -c tmp/airports.dat > data/airports.dat.gz
+
+data/countries.dat.gz: $(DIRS)
+	wget -c https://raw.githubusercontent.com/jpatokal/openflights/master/data/countries.dat -O tmp/countries.dat && \
+  gzip -c tmp/countries.dat > data/countries.dat.gz
+
+# IGNORE
+# See https://unece.org/trade/cefact/UNLOCODE-Download
+tmp/locode.csv.zip :$(DIRS)
+	wget -c https://service.unece.org/trade/locode/loc222csv.zip -O tmp/locode.csv.zip
+
+data/unlocode-SubdivisionCodes.csv.gz: tmp/locode.csv.zip
+	unzip -p tmp/locode.csv.zip "*SubdivisionCodes.csv" | gzip > data/unlocode-SubdivisionCodes.csv.gz
+
+data/unlocode-CodeList.csv.gz:  tmp/locode.csv.zip
+	unzip -p tmp/locode.csv.zip "*CodeListPart*" | gzip > data/unlocode-CodeList.csv.gz
+
+# Download and repack external data-sources
+update: data/unlocode-CodeList.csv.gz data/unlocode-SubdivisionCodes.csv.gz data/countries.dat.gz data/airports.dat.gz
+
 # Print version
 version:
 	@echo $(VERSION)
@@ -58,6 +88,14 @@ help:
 	@echo ''
 	@awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print "   ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u
 
+
+# Clean data directory before update
+clean_data:
+	rm -f data/unlocode-CodeList.csv.gz
+	rm -f data/unlocode-SubdivisionCodes.csv.gz
+	rm -f data/countries.dat.gz
+	rm -f data/airports.dat.gz
+
 # Clean up
 clean:
 	rm -f in/*
@@ -74,5 +112,6 @@ debpackage:
 		"Please see CHANGELOG.md for code changes for $(VERSION)"
 	dpkg-buildpackage --no-sign -b
 
+# Clean up debian packaging leftovers
 debclean:
 	dh clean