Compare commits
3
Commits
077b44ece1
...
b54020e23b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b54020e23b | ||
|
|
2878270187 | ||
|
|
007dee7341 |
No files matched your search
@@ -12,6 +12,10 @@
|
|||||||
# see docs/development.adoc)
|
# see docs/development.adoc)
|
||||||
/data/*.pmtiles
|
/data/*.pmtiles
|
||||||
|
|
||||||
|
# Per-country .osm.pbf cache for offline.pmtiles rebuilds (also large,
|
||||||
|
# also local-only — see PmtilesBuilder.cached_extract)
|
||||||
|
/data/geofabrik_cache/
|
||||||
|
|
||||||
# Offline Wikipedia ZIM for kiwix-serve (also large, also local-only —
|
# Offline Wikipedia ZIM for kiwix-serve (also large, also local-only —
|
||||||
# same story, see docs/development.adoc)
|
# same story, see docs/development.adoc)
|
||||||
/data/kiwix/*
|
/data/kiwix/*
|
||||||
|
|||||||
@@ -277,6 +277,33 @@ module Maptool
|
|||||||
json PmtilesBuilder.offline_status
|
json PmtilesBuilder.offline_status
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Deletes offline.pmtiles plus its whole per-country download cache —
|
||||||
|
# a manual "reclaim the space"/"start over" action, distinct from
|
||||||
|
# unticking countries in the region picker (which only changes what a
|
||||||
|
# future build covers, doesn't free anything already on disk).
|
||||||
|
delete "/api/offline-tiles" do
|
||||||
|
require_login!
|
||||||
|
begin
|
||||||
|
PmtilesBuilder.delete_offline_data(DATA_DIR)
|
||||||
|
rescue PmtilesBuilder::BuildError => e
|
||||||
|
halt 409, json(error: e.message)
|
||||||
|
end
|
||||||
|
json ok: true
|
||||||
|
end
|
||||||
|
|
||||||
|
# Deletes one country's cached Geofabrik extract only — e.g. to force a
|
||||||
|
# clean re-download of just that one, or reclaim space for a country
|
||||||
|
# that's no longer selected (the cache otherwise keeps it indefinitely).
|
||||||
|
delete "/api/offline-tiles/cache/:code" do
|
||||||
|
require_login!
|
||||||
|
begin
|
||||||
|
PmtilesBuilder.delete_cached_extract(DATA_DIR, params[:code])
|
||||||
|
rescue PmtilesBuilder::BuildError => e
|
||||||
|
halt 409, json(error: e.message)
|
||||||
|
end
|
||||||
|
json ok: true
|
||||||
|
end
|
||||||
|
|
||||||
# Same optional-local-file story as offline.pmtiles above — see
|
# Same optional-local-file story as offline.pmtiles above — see
|
||||||
# docs/development.adoc for how to generate one, until then hillshade
|
# docs/development.adoc for how to generate one, until then hillshade
|
||||||
# just stays unavailable while the offline basemap is active.
|
# just stays unavailable while the offline basemap is active.
|
||||||
@@ -309,6 +336,17 @@ module Maptool
|
|||||||
json PmtilesBuilder.terrain_status
|
json PmtilesBuilder.terrain_status
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Same reasoning as DELETE /api/offline-tiles above.
|
||||||
|
delete "/api/terrain-tiles" do
|
||||||
|
require_login!
|
||||||
|
begin
|
||||||
|
PmtilesBuilder.delete_terrain_data(DATA_DIR)
|
||||||
|
rescue PmtilesBuilder::BuildError => e
|
||||||
|
halt 409, json(error: e.message)
|
||||||
|
end
|
||||||
|
json ok: true
|
||||||
|
end
|
||||||
|
|
||||||
# Pinged once per page load (public/js/map.js) — url is a same-origin
|
# Pinged once per page load (public/js/map.js) — url is a same-origin
|
||||||
# path (not kiwix-serve's own address, which the browser can't reach
|
# path (not kiwix-serve's own address, which the browser can't reach
|
||||||
# directly behind the reverse proxy in front of this app), proxied by
|
# directly behind the reverse proxy in front of this app), proxied by
|
||||||
|
|||||||
+29
-10
@@ -94,15 +94,20 @@ Ticking countries and clicking "Apply changes" saves the selection
|
|||||||
(`data/offline_regions.json`, an array of ISO A2 codes) and rebuilds
|
(`data/offline_regions.json`, an array of ISO A2 codes) and rebuilds
|
||||||
both files to cover it.
|
both files to cover it.
|
||||||
|
|
||||||
There's no standalone per-country file to individually delete —
|
There's no standalone per-country *built* file — `offline.pmtiles`/
|
||||||
`offline.pmtiles`/`terrain.pmtiles` are always one combined extract for
|
`terrain.pmtiles` are always one combined extract for the *whole
|
||||||
the *whole current selection*. "Remove a country" and "refresh a
|
current selection*. "Remove a country" is just "change the selection,
|
||||||
country" are both just "change the selection, then Apply" (unticking a
|
then Apply" (unticking a country and applying rebuilds without it).
|
||||||
country and applying rebuilds without it; nothing to separately
|
terrain.pmtiles re-extracts its selection from scratch on
|
||||||
delete). Every Apply re-extracts the entire selection from scratch, not
|
every Apply — fine given that extraction is byte-range requests
|
||||||
just what changed — fine given extraction is fast (byte-range requests
|
against a remote archive, not full downloads. offline.pmtiles, by
|
||||||
against remote archives, not per-tile fetches), but worth revisiting if
|
contrast, downloads a whole country's `.osm.pbf` from Geofabrik (can be
|
||||||
someone selects half of Europe at once and finds it slow.
|
multi-GB), so it keeps a per-country cache under
|
||||||
|
`data/geofabrik_cache/` instead of re-fetching on every Apply — a
|
||||||
|
country already cached is only re-downloaded if Geofabrik's own
|
||||||
|
`Last-Modified` for it has moved on since (checked with a cheap `HEAD`,
|
||||||
|
see `PmtilesBuilder.cached_extract`), so growing or shrinking the
|
||||||
|
selection only touches whatever's actually new to it.
|
||||||
|
|
||||||
This wraps exactly the manual workflows documented above
|
This wraps exactly the manual workflows documented above
|
||||||
(`lib/maptool/pmtiles_builder.rb`). offline.pmtiles downloads each
|
(`lib/maptool/pmtiles_builder.rb`). offline.pmtiles downloads each
|
||||||
@@ -128,7 +133,21 @@ the current selection, `POST` to change it, then `POST
|
|||||||
/api/offline-tiles/build` / `/api/terrain-tiles/build`, polled via the
|
/api/offline-tiles/build` / `/api/terrain-tiles/build`, polled via the
|
||||||
matching `*/build-status` route) — only one of each kind at a time;
|
matching `*/build-status` route) — only one of each kind at a time;
|
||||||
starting a second while one's running gets a 409, as does starting
|
starting a second while one's running gets a 409, as does starting
|
||||||
either with nothing selected yet.
|
either with nothing selected yet. While running, `build-status` also
|
||||||
|
includes a `step` string (e.g. `"fetching PL (3/4) — done: SK, CZ —
|
||||||
|
pending: HU"`, or `"tiling with planetiler"`) — see
|
||||||
|
`PmtilesBuilder.run_job`'s `report_step`.
|
||||||
|
|
||||||
|
`DELETE /api/offline-tiles` removes offline.pmtiles, its sidecar meta,
|
||||||
|
and the whole per-country download cache (`data/geofabrik_cache/`) —
|
||||||
|
`DELETE /api/terrain-tiles` does the same for terrain.pmtiles. Both are
|
||||||
|
a manual "reclaim the space" / "start over" action (409 while that
|
||||||
|
kind's build is running), separate from the region picker's Apply,
|
||||||
|
which only changes what a *future* build covers and never frees
|
||||||
|
anything already on disk. `DELETE
|
||||||
|
/api/offline-tiles/cache/:code` removes just one country's cached
|
||||||
|
extract, e.g. to force a clean re-download of it without touching the
|
||||||
|
rest of the cache.
|
||||||
|
|
||||||
Each successful build also writes a small sidecar (`offline.pmtiles.
|
Each successful build also writes a small sidecar (`offline.pmtiles.
|
||||||
meta.json` — each source country's code/Geofabrik slug/Last-Modified;
|
meta.json` — each source country's code/Geofabrik slug/Last-Modified;
|
||||||
|
|||||||
+146
-24
@@ -179,17 +179,21 @@ module Maptool
|
|||||||
# finishes — raises BuildError (the caller turns this into an error
|
# finishes — raises BuildError (the caller turns this into an error
|
||||||
# response) if nothing's selected yet, or one's already running.
|
# response) if nothing's selected yet, or one's already running.
|
||||||
#
|
#
|
||||||
# download_extract exists as an injectable seam so tests can exercise
|
# download_extract/fetch_last_modified exist as injectable seams so
|
||||||
# the rest of this (scratch dir, osmium/planetiler invocation, atomic
|
# tests can exercise the rest of this (scratch dir, osmium/planetiler
|
||||||
# rename) without a real network call — same reasoning test_helper.rb
|
# invocation, atomic rename, the per-country cache below) without a
|
||||||
# already gives for keeping /api/plugins route tests off the real
|
# real network call — same reasoning test_helper.rb already gives for
|
||||||
# Overpass API.
|
# keeping /api/plugins route tests off the real Overpass API.
|
||||||
def self.build_offline(data_dir:, download_extract: method(:download_extract))
|
def self.build_offline(data_dir:, download_extract: method(:download_extract), fetch_last_modified: method(:fetch_geofabrik_last_modified))
|
||||||
codes = selected_regions(data_dir)
|
codes = selected_regions(data_dir)
|
||||||
raise BuildError, "select at least one region first" if codes.empty?
|
raise BuildError, "select at least one region first" if codes.empty?
|
||||||
|
|
||||||
begin_run!(OFFLINE_MUTEX, OFFLINE_STATUS)
|
begin_run!(OFFLINE_MUTEX, OFFLINE_STATUS)
|
||||||
Thread.new { run_job(OFFLINE_MUTEX, OFFLINE_STATUS) { extract_offline(codes, data_dir, download_extract) } }
|
Thread.new do
|
||||||
|
run_job(OFFLINE_MUTEX, OFFLINE_STATUS) do |report_step|
|
||||||
|
extract_offline(codes, data_dir, download_extract, fetch_last_modified, report_step)
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# fetch_manifest: same injectable-seam reasoning as download_extract
|
# fetch_manifest: same injectable-seam reasoning as download_extract
|
||||||
@@ -199,7 +203,11 @@ module Maptool
|
|||||||
raise BuildError, "select at least one region first" if codes.empty?
|
raise BuildError, "select at least one region first" if codes.empty?
|
||||||
|
|
||||||
begin_run!(TERRAIN_MUTEX, TERRAIN_STATUS)
|
begin_run!(TERRAIN_MUTEX, TERRAIN_STATUS)
|
||||||
Thread.new { run_job(TERRAIN_MUTEX, TERRAIN_STATUS) { build_terrain_pmtiles(codes, data_dir, fetch_manifest) } }
|
Thread.new do
|
||||||
|
run_job(TERRAIN_MUTEX, TERRAIN_STATUS) do |report_step|
|
||||||
|
build_terrain_pmtiles(codes, data_dir, fetch_manifest, report_step)
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.begin_run!(mutex, status)
|
def self.begin_run!(mutex, status)
|
||||||
@@ -210,11 +218,19 @@ module Maptool
|
|||||||
end
|
end
|
||||||
private_class_method :begin_run!
|
private_class_method :begin_run!
|
||||||
|
|
||||||
|
# Yields a report_step proc the build can call with a short
|
||||||
|
# human-readable description of what it's doing right now ("downloading
|
||||||
|
# SK (1/2)", "tiling with planetiler", ...) — surfaced as status[:step]
|
||||||
|
# so GET .../build-status can show progress beyond just "running since
|
||||||
|
# X" (see docs/development.adoc).
|
||||||
def self.run_job(mutex, status)
|
def self.run_job(mutex, status)
|
||||||
yield
|
report_step = ->(text) { mutex.synchronize { status[:step] = text } }
|
||||||
|
yield report_step
|
||||||
mutex.synchronize { status.merge!(state: "done", finished_at: Time.now.utc.iso8601).delete(:message) }
|
mutex.synchronize { status.merge!(state: "done", finished_at: Time.now.utc.iso8601).delete(:message) }
|
||||||
rescue => e
|
rescue => e
|
||||||
mutex.synchronize { status.merge!(state: "error", finished_at: Time.now.utc.iso8601, message: e.message) }
|
mutex.synchronize { status.merge!(state: "error", finished_at: Time.now.utc.iso8601, message: e.message) }
|
||||||
|
ensure
|
||||||
|
mutex.synchronize { status.delete(:step) }
|
||||||
end
|
end
|
||||||
private_class_method :run_job
|
private_class_method :run_job
|
||||||
|
|
||||||
@@ -228,25 +244,30 @@ module Maptool
|
|||||||
# tiles.openfreemap.org's Liberty style already expects, so
|
# tiles.openfreemap.org's Liberty style already expects, so
|
||||||
# public/js/map.js's offlineStyle() can reuse ONLINE_STYLE_PATCH
|
# public/js/map.js's offlineStyle() can reuse ONLINE_STYLE_PATCH
|
||||||
# unchanged instead of a separate protomaps-themes-base palette.
|
# unchanged instead of a separate protomaps-themes-base palette.
|
||||||
def self.extract_offline(codes, data_dir, download_extract)
|
def self.extract_offline(codes, data_dir, download_extract, fetch_last_modified, report_step)
|
||||||
scratch_dir = Dir.mktmpdir("maptool-offline")
|
scratch_dir = Dir.mktmpdir("maptool-offline")
|
||||||
sources = codes.map do |code|
|
sources = codes.each_with_index.map do |code, i|
|
||||||
slug = GEOFABRIK_SLUGS.fetch(code) { raise BuildError, "no Geofabrik source known for region #{code}" }
|
slug = GEOFABRIK_SLUGS.fetch(code) { raise BuildError, "no Geofabrik source known for region #{code}" }
|
||||||
dest = File.join(scratch_dir, "#{code}.osm.pbf")
|
report_step.call(progress_step("fetching #{code}", codes, i))
|
||||||
last_modified = download_extract.call(geofabrik_url(slug), dest)
|
cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
|
||||||
{ "code" => code, "slug" => slug, "last_modified" => last_modified, "path" => dest }
|
|
||||||
end
|
end
|
||||||
|
|
||||||
osm_path =
|
osm_path =
|
||||||
if sources.size == 1
|
if sources.size == 1
|
||||||
sources.first["path"]
|
sources.first["path"]
|
||||||
else
|
else
|
||||||
|
report_step.call("merging with osmium")
|
||||||
merged_path = File.join(scratch_dir, "merged.osm.pbf")
|
merged_path = File.join(scratch_dir, "merged.osm.pbf")
|
||||||
run_osmium!("merge", *sources.map { |s| s["path"] }, "-o", merged_path)
|
run_osmium!("merge", *sources.map { |s| s["path"] }, "-o", merged_path)
|
||||||
merged_path
|
merged_path
|
||||||
end
|
end
|
||||||
|
|
||||||
tmp_path = File.join(data_dir, "offline.pmtiles.tmp")
|
report_step.call("tiling with planetiler")
|
||||||
|
# .tmp.pmtiles, not .pmtiles.tmp — Planetiler picks its output archive
|
||||||
|
# format (pmtiles vs mbtiles) from the file extension, so a ".tmp"
|
||||||
|
# suffix made it fail with "Unsupported format tmp" instead of
|
||||||
|
# actually tiling anything.
|
||||||
|
tmp_path = File.join(data_dir, "offline.tmp.pmtiles")
|
||||||
run_planetiler!("--osm-path=#{osm_path}", "--output=#{tmp_path}")
|
run_planetiler!("--osm-path=#{osm_path}", "--output=#{tmp_path}")
|
||||||
File.rename(tmp_path, File.join(data_dir, "offline.pmtiles"))
|
File.rename(tmp_path, File.join(data_dir, "offline.pmtiles"))
|
||||||
File.write(offline_meta_path(data_dir), JSON.generate(sources: sources.map { |s| s.slice("code", "slug", "last_modified") }))
|
File.write(offline_meta_path(data_dir), JSON.generate(sources: sources.map { |s| s.slice("code", "slug", "last_modified") }))
|
||||||
@@ -256,11 +277,60 @@ module Maptool
|
|||||||
end
|
end
|
||||||
private_class_method :extract_offline
|
private_class_method :extract_offline
|
||||||
|
|
||||||
|
# Renders a report_step string like "fetching PL (3/4) — done: SK, CZ —
|
||||||
|
# pending: HU" — so GET .../build-status shows not just "it's running"
|
||||||
|
# but which of the current build's sources is in flight and which are
|
||||||
|
# already behind/ahead of it.
|
||||||
|
def self.progress_step(action, labels, index)
|
||||||
|
done = labels[0...index]
|
||||||
|
pending = labels[(index + 1)..] || []
|
||||||
|
parts = ["#{action} (#{index + 1}/#{labels.size})"]
|
||||||
|
parts << "done: #{done.join(", ")}" unless done.empty?
|
||||||
|
parts << "pending: #{pending.join(", ")}" unless pending.empty?
|
||||||
|
parts.join(" — ")
|
||||||
|
end
|
||||||
|
private_class_method :progress_step
|
||||||
|
|
||||||
def self.geofabrik_url(slug)
|
def self.geofabrik_url(slug)
|
||||||
"https://download.geofabrik.de/#{slug}-latest.osm.pbf"
|
"https://download.geofabrik.de/#{slug}-latest.osm.pbf"
|
||||||
end
|
end
|
||||||
private_class_method :geofabrik_url
|
private_class_method :geofabrik_url
|
||||||
|
|
||||||
|
# Per-country .osm.pbf cache, kept across builds under data_dir (not
|
||||||
|
# scratch_dir, which is wiped every run) — changing the region
|
||||||
|
# selection re-downloads only the countries that are new to it, and
|
||||||
|
# re-running the same selection re-downloads nothing unless Geofabrik's
|
||||||
|
# own Last-Modified for that country moved on. A stamp file next to
|
||||||
|
# each cached extract records the Last-Modified it was fetched with;
|
||||||
|
# a HEAD check (cheap) decides whether the multi-GB GET (not cheap) is
|
||||||
|
# skippable.
|
||||||
|
def self.cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
|
||||||
|
cache_dir = geofabrik_cache_dir(data_dir)
|
||||||
|
FileUtils.mkdir_p(cache_dir)
|
||||||
|
path = File.join(cache_dir, "#{code}.osm.pbf")
|
||||||
|
stamp_path = "#{path}.last_modified"
|
||||||
|
url = geofabrik_url(slug)
|
||||||
|
|
||||||
|
if File.exist?(path) && File.exist?(stamp_path)
|
||||||
|
current = begin
|
||||||
|
fetch_last_modified.call(url)
|
||||||
|
rescue BuildError
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
return { "code" => code, "slug" => slug, "last_modified" => current, "path" => path } if current && current == File.read(stamp_path)
|
||||||
|
end
|
||||||
|
|
||||||
|
last_modified = download_extract.call(url, path)
|
||||||
|
File.write(stamp_path, last_modified.to_s)
|
||||||
|
{ "code" => code, "slug" => slug, "last_modified" => last_modified, "path" => path }
|
||||||
|
end
|
||||||
|
private_class_method :cached_extract
|
||||||
|
|
||||||
|
def self.geofabrik_cache_dir(data_dir)
|
||||||
|
File.join(data_dir, "geofabrik_cache")
|
||||||
|
end
|
||||||
|
private_class_method :geofabrik_cache_dir
|
||||||
|
|
||||||
# Real network implementation of the download_extract seam above —
|
# Real network implementation of the download_extract seam above —
|
||||||
# streams the extract to dest (some of these are multi-GB — Net::HTTP's
|
# streams the extract to dest (some of these are multi-GB — Net::HTTP's
|
||||||
# plain #get buffers the whole response body in memory first, which
|
# plain #get buffers the whole response body in memory first, which
|
||||||
@@ -345,6 +415,31 @@ module Maptool
|
|||||||
end
|
end
|
||||||
private_class_method :offline_meta_path
|
private_class_method :offline_meta_path
|
||||||
|
|
||||||
|
# Deletes offline.pmtiles, its sidecar meta, and the whole per-country
|
||||||
|
# download cache (data/geofabrik_cache/) — "start over" for the offline
|
||||||
|
# basemap. The current selection (data/offline_regions.json) is left
|
||||||
|
# alone, so a following build just re-downloads it fresh.
|
||||||
|
def self.delete_offline_data(data_dir)
|
||||||
|
OFFLINE_MUTEX.synchronize { raise BuildError, "a build is already running" if OFFLINE_STATUS[:state] == "running" }
|
||||||
|
|
||||||
|
File.delete(File.join(data_dir, "offline.pmtiles")) if File.exist?(File.join(data_dir, "offline.pmtiles"))
|
||||||
|
File.delete(offline_meta_path(data_dir)) if File.exist?(offline_meta_path(data_dir))
|
||||||
|
cache_dir = geofabrik_cache_dir(data_dir)
|
||||||
|
FileUtils.remove_entry(cache_dir) if Dir.exist?(cache_dir)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Deletes just one country's cached .osm.pbf (and its Last-Modified
|
||||||
|
# stamp) — e.g. to force a fresh download without touching the others,
|
||||||
|
# or to reclaim space for a country no longer selected.
|
||||||
|
def self.delete_cached_extract(data_dir, code)
|
||||||
|
OFFLINE_MUTEX.synchronize { raise BuildError, "a build is already running" if OFFLINE_STATUS[:state] == "running" }
|
||||||
|
|
||||||
|
cache_dir = geofabrik_cache_dir(data_dir)
|
||||||
|
path = File.join(cache_dir, "#{code}.osm.pbf")
|
||||||
|
File.delete(path) if File.exist?(path)
|
||||||
|
File.delete("#{path}.last_modified") if File.exist?("#{path}.last_modified")
|
||||||
|
end
|
||||||
|
|
||||||
# --- terrain.pmtiles ---
|
# --- terrain.pmtiles ---
|
||||||
|
|
||||||
# planet.pmtiles (z0-12, always) plus whichever regional chunks
|
# planet.pmtiles (z0-12, always) plus whichever regional chunks
|
||||||
@@ -361,7 +456,8 @@ module Maptool
|
|||||||
# border tiles (extraction copies whole tiles, it doesn't clip vector
|
# border tiles (extraction copies whole tiles, it doesn't clip vector
|
||||||
# content to the polygon) — so this always does one extract per
|
# content to the polygon) — so this always does one extract per
|
||||||
# *source archive*, never one per country.
|
# *source archive*, never one per country.
|
||||||
def self.build_terrain_pmtiles(codes, data_dir, fetch_manifest)
|
def self.build_terrain_pmtiles(codes, data_dir, fetch_manifest, report_step)
|
||||||
|
report_step.call("fetching mapterhorn manifest")
|
||||||
items = fetch_manifest.call
|
items = fetch_manifest.call
|
||||||
planet = items.find { |item| item["name"] == MAPTERHORN_PLANET_NAME }
|
planet = items.find { |item| item["name"] == MAPTERHORN_PLANET_NAME }
|
||||||
raise BuildError, "mapterhorn manifest is missing #{MAPTERHORN_PLANET_NAME}" unless planet
|
raise BuildError, "mapterhorn manifest is missing #{MAPTERHORN_PLANET_NAME}" unless planet
|
||||||
@@ -374,8 +470,14 @@ module Maptool
|
|||||||
|
|
||||||
chunks = items.select { |item| item["name"] != MAPTERHORN_PLANET_NAME && bbox_intersects?(item, south, west, north, east) }
|
chunks = items.select { |item| item["name"] != MAPTERHORN_PLANET_NAME && bbox_intersects?(item, south, west, north, east) }
|
||||||
|
|
||||||
tmp_path = File.join(data_dir, "terrain.pmtiles.tmp")
|
# .tmp.pmtiles, not .pmtiles.tmp — same reasoning as offline.pmtiles'
|
||||||
extracts = [planet, *chunks].each_with_index.map do |item, i|
|
# own tmp_path above, in case go-pmtiles also derives format from
|
||||||
|
# the extension for its "merge" output.
|
||||||
|
tmp_path = File.join(data_dir, "terrain.tmp.pmtiles")
|
||||||
|
sources = [planet, *chunks]
|
||||||
|
labels = sources.map { |item| item["name"] }
|
||||||
|
extracts = sources.each_with_index.map do |item, i|
|
||||||
|
report_step.call(progress_step("extracting #{item["name"]}", labels, i))
|
||||||
extract_path = File.join(scratch_dir, "extract-#{i}.pmtiles")
|
extract_path = File.join(scratch_dir, "extract-#{i}.pmtiles")
|
||||||
run_pmtiles!("extract", item["url"], extract_path, "--region=#{region_path}", "--maxzoom=#{TERRAIN_MAX_ZOOM}")
|
run_pmtiles!("extract", item["url"], extract_path, "--region=#{region_path}", "--maxzoom=#{TERRAIN_MAX_ZOOM}")
|
||||||
extract_path
|
extract_path
|
||||||
@@ -384,6 +486,7 @@ module Maptool
|
|||||||
if extracts.size == 1
|
if extracts.size == 1
|
||||||
File.rename(extracts.first, tmp_path)
|
File.rename(extracts.first, tmp_path)
|
||||||
else
|
else
|
||||||
|
report_step.call("merging with pmtiles")
|
||||||
run_pmtiles!("merge", *extracts, tmp_path)
|
run_pmtiles!("merge", *extracts, tmp_path)
|
||||||
end
|
end
|
||||||
File.rename(tmp_path, File.join(data_dir, "terrain.pmtiles"))
|
File.rename(tmp_path, File.join(data_dir, "terrain.pmtiles"))
|
||||||
@@ -446,6 +549,17 @@ module Maptool
|
|||||||
end
|
end
|
||||||
private_class_method :terrain_meta_path
|
private_class_method :terrain_meta_path
|
||||||
|
|
||||||
|
# Deletes terrain.pmtiles and its sidecar meta — same "start over"
|
||||||
|
# reasoning as delete_offline_data above. No download cache to clean up
|
||||||
|
# here: terrain.pmtiles' pmtiles-extract step reads its sources over
|
||||||
|
# HTTP range requests rather than downloading a whole archive first.
|
||||||
|
def self.delete_terrain_data(data_dir)
|
||||||
|
TERRAIN_MUTEX.synchronize { raise BuildError, "a build is already running" if TERRAIN_STATUS[:state] == "running" }
|
||||||
|
|
||||||
|
File.delete(File.join(data_dir, "terrain.pmtiles")) if File.exist?(File.join(data_dir, "terrain.pmtiles"))
|
||||||
|
File.delete(terrain_meta_path(data_dir)) if File.exist?(terrain_meta_path(data_dir))
|
||||||
|
end
|
||||||
|
|
||||||
# --- shared ---
|
# --- shared ---
|
||||||
|
|
||||||
def self.read_json(path)
|
def self.read_json(path)
|
||||||
@@ -473,25 +587,33 @@ module Maptool
|
|||||||
ENV.fetch("PLANETILER_JAR", "/usr/local/lib/planetiler.jar")
|
ENV.fetch("PLANETILER_JAR", "/usr/local/lib/planetiler.jar")
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Some CLI tools (go-pmtiles included) write their actual error to
|
||||||
|
# stdout rather than stderr — combine both rather than risk raising a
|
||||||
|
# BuildError with nothing useful in it.
|
||||||
|
def self.command_output(stdout, stderr)
|
||||||
|
[stdout, stderr].map(&:strip).reject(&:empty?).join("\n")
|
||||||
|
end
|
||||||
|
private_class_method :command_output
|
||||||
|
|
||||||
def self.run_pmtiles!(*args)
|
def self.run_pmtiles!(*args)
|
||||||
_stdout, stderr, status = Open3.capture3(pmtiles_bin, *args)
|
stdout, stderr, status = Open3.capture3(pmtiles_bin, *args)
|
||||||
raise BuildError, "pmtiles exited with status #{status.exitstatus}: #{stderr.strip}" unless status.success?
|
raise BuildError, "pmtiles exited with status #{status.exitstatus}: #{command_output(stdout, stderr)}" unless status.success?
|
||||||
rescue Errno::ENOENT, Errno::EACCES => e
|
rescue Errno::ENOENT, Errno::EACCES => e
|
||||||
raise BuildError, "could not run '#{pmtiles_bin}' (#{e.message}) — is go-pmtiles installed and on PATH? See docs/development.adoc."
|
raise BuildError, "could not run '#{pmtiles_bin}' (#{e.message}) — is go-pmtiles installed and on PATH? See docs/development.adoc."
|
||||||
end
|
end
|
||||||
private_class_method :run_pmtiles!
|
private_class_method :run_pmtiles!
|
||||||
|
|
||||||
def self.run_osmium!(*args)
|
def self.run_osmium!(*args)
|
||||||
_stdout, stderr, status = Open3.capture3(osmium_bin, *args)
|
stdout, stderr, status = Open3.capture3(osmium_bin, *args)
|
||||||
raise BuildError, "osmium exited with status #{status.exitstatus}: #{stderr.strip}" unless status.success?
|
raise BuildError, "osmium exited with status #{status.exitstatus}: #{command_output(stdout, stderr)}" unless status.success?
|
||||||
rescue Errno::ENOENT, Errno::EACCES => e
|
rescue Errno::ENOENT, Errno::EACCES => e
|
||||||
raise BuildError, "could not run '#{osmium_bin}' (#{e.message}) — is osmium-tool installed and on PATH? See docs/development.adoc."
|
raise BuildError, "could not run '#{osmium_bin}' (#{e.message}) — is osmium-tool installed and on PATH? See docs/development.adoc."
|
||||||
end
|
end
|
||||||
private_class_method :run_osmium!
|
private_class_method :run_osmium!
|
||||||
|
|
||||||
def self.run_planetiler!(*args)
|
def self.run_planetiler!(*args)
|
||||||
_stdout, stderr, status = Open3.capture3(java_bin, "-jar", planetiler_jar, *args)
|
stdout, stderr, status = Open3.capture3(java_bin, "-jar", planetiler_jar, *args)
|
||||||
raise BuildError, "planetiler exited with status #{status.exitstatus}: #{stderr.strip}" unless status.success?
|
raise BuildError, "planetiler exited with status #{status.exitstatus}: #{command_output(stdout, stderr)}" unless status.success?
|
||||||
rescue Errno::ENOENT, Errno::EACCES => e
|
rescue Errno::ENOENT, Errno::EACCES => e
|
||||||
raise BuildError, "could not run '#{java_bin} -jar #{planetiler_jar}' (#{e.message}) — is Java and planetiler.jar installed? See docs/development.adoc."
|
raise BuildError, "could not run '#{java_bin} -jar #{planetiler_jar}' (#{e.message}) — is Java and planetiler.jar installed? See docs/development.adoc."
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -472,6 +472,33 @@ html, body {
|
|||||||
border-top: 1px solid #eee;
|
border-top: 1px solid #eee;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.offline-data-panel__row-head {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.offline-data-panel__delete {
|
||||||
|
border: 1px solid #ccc;
|
||||||
|
border-radius: 4px;
|
||||||
|
background: #fff;
|
||||||
|
padding: 2px 8px;
|
||||||
|
font: inherit;
|
||||||
|
font-size: 11px;
|
||||||
|
color: #a33;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.offline-data-panel__delete:hover {
|
||||||
|
background: #fdecec;
|
||||||
|
}
|
||||||
|
|
||||||
|
.offline-data-panel__delete:disabled {
|
||||||
|
opacity: 0.5;
|
||||||
|
cursor: default;
|
||||||
|
}
|
||||||
|
|
||||||
.offline-data-panel__status {
|
.offline-data-panel__status {
|
||||||
margin: 6px 0 0;
|
margin: 6px 0 0;
|
||||||
color: #666;
|
color: #666;
|
||||||
|
|||||||
@@ -17,12 +17,14 @@ function formatElapsed(ms) {
|
|||||||
return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`;
|
return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`;
|
||||||
}
|
}
|
||||||
|
|
||||||
// "Building…" alone gave no way to tell a slow build from a stuck one —
|
// "Building…" alone gave no way to tell a slow build from a stuck one, or
|
||||||
// elapsed time answers "is it alive" regardless of how long a given
|
// which of the current selection's sources it's actually working on right
|
||||||
// selection takes.
|
// now — elapsed time answers "is it alive", status.step (see
|
||||||
|
// PmtilesBuilder.run_job) answers "doing what, and what's left".
|
||||||
function formatRunning(status) {
|
function formatRunning(status) {
|
||||||
const elapsed = status.started_at ? ` (${formatElapsed(Date.now() - new Date(status.started_at).getTime())})` : "";
|
const elapsed = status.started_at ? ` (${formatElapsed(Date.now() - new Date(status.started_at).getTime())})` : "";
|
||||||
return `Building…${elapsed}`;
|
const step = status.step ? `: ${status.step}` : "";
|
||||||
|
return `Building…${elapsed}${step}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Vendored square-cropped SVGs (public/vendor/flag-icons/, MIT, see
|
// Vendored square-cropped SVGs (public/vendor/flag-icons/, MIT, see
|
||||||
@@ -131,9 +133,10 @@ class RegionPicker {
|
|||||||
// one class covers both rather than writing the same fetch/poll logic
|
// one class covers both rather than writing the same fetch/poll logic
|
||||||
// twice.
|
// twice.
|
||||||
class DataFileRow {
|
class DataFileRow {
|
||||||
constructor({ label, statusUrl, buildUrl, buildStatusUrl, checkUpdateUrl, formatMeta }) {
|
constructor({ label, statusUrl, deleteUrl, buildUrl, buildStatusUrl, checkUpdateUrl, formatMeta }) {
|
||||||
this._label = label;
|
this._label = label;
|
||||||
this._statusUrl = statusUrl;
|
this._statusUrl = statusUrl;
|
||||||
|
this._deleteUrl = deleteUrl;
|
||||||
this._buildUrl = buildUrl;
|
this._buildUrl = buildUrl;
|
||||||
this._buildStatusUrl = buildStatusUrl;
|
this._buildStatusUrl = buildStatusUrl;
|
||||||
this._checkUpdateUrl = checkUpdateUrl;
|
this._checkUpdateUrl = checkUpdateUrl;
|
||||||
@@ -144,10 +147,15 @@ class DataFileRow {
|
|||||||
const row = document.createElement("div");
|
const row = document.createElement("div");
|
||||||
row.className = "offline-data-panel__row";
|
row.className = "offline-data-panel__row";
|
||||||
row.innerHTML = `
|
row.innerHTML = `
|
||||||
<strong>${this._label}</strong>
|
<div class="offline-data-panel__row-head">
|
||||||
|
<strong>${this._label}</strong>
|
||||||
|
<button type="button" class="offline-data-panel__delete" hidden>Delete</button>
|
||||||
|
</div>
|
||||||
<p class="offline-data-panel__status">Checking…</p>
|
<p class="offline-data-panel__status">Checking…</p>
|
||||||
`;
|
`;
|
||||||
this._statusEl = row.querySelector(".offline-data-panel__status");
|
this._statusEl = row.querySelector(".offline-data-panel__status");
|
||||||
|
this._deleteButton = row.querySelector(".offline-data-panel__delete");
|
||||||
|
this._deleteButton.addEventListener("click", () => this._delete());
|
||||||
container.appendChild(row);
|
container.appendChild(row);
|
||||||
this._refreshStatus();
|
this._refreshStatus();
|
||||||
}
|
}
|
||||||
@@ -163,14 +171,39 @@ class DataFileRow {
|
|||||||
|
|
||||||
if (!data.available) {
|
if (!data.available) {
|
||||||
this._statusEl.textContent = "Not generated yet.";
|
this._statusEl.textContent = "Not generated yet.";
|
||||||
|
this._deleteButton.hidden = true;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const updated = new Date(data.mtime).toLocaleString([], { dateStyle: "short", timeStyle: "short" });
|
const updated = new Date(data.mtime).toLocaleString([], { dateStyle: "short", timeStyle: "short" });
|
||||||
this._statusEl.textContent = `${formatBytes(data.size)} · updated ${updated}${this._formatMeta(data.meta)}`;
|
this._statusEl.textContent = `${formatBytes(data.size)} · updated ${updated}${this._formatMeta(data.meta)}`;
|
||||||
|
this._deleteButton.hidden = !isLoggedIn();
|
||||||
this._checkUpdate();
|
this._checkUpdate();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Deletes the built file (and, for offline.pmtiles, its whole per-country
|
||||||
|
// download cache — see PmtilesBuilder.delete_offline_data) so the row
|
||||||
|
// goes back to "Not generated yet." — a manual way to reclaim space or
|
||||||
|
// force a completely fresh rebuild, since Apply alone never removes
|
||||||
|
// anything already on disk.
|
||||||
|
async _delete() {
|
||||||
|
if (!window.confirm(`Delete ${this._label.toLowerCase()}? This can't be undone.`)) return;
|
||||||
|
|
||||||
|
this._deleteButton.disabled = true;
|
||||||
|
try {
|
||||||
|
const response = await fetch(this._deleteUrl, { method: "DELETE" });
|
||||||
|
const data = await response.json().catch(() => ({}));
|
||||||
|
if (!response.ok) throw new Error(data.error || `could not delete: ${response.status}`);
|
||||||
|
} catch (err) {
|
||||||
|
window.alert(err.message);
|
||||||
|
return;
|
||||||
|
} finally {
|
||||||
|
this._deleteButton.disabled = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
this._refreshStatus();
|
||||||
|
}
|
||||||
|
|
||||||
// Separate from _refreshStatus's own fetch (and only called once, right
|
// Separate from _refreshStatus's own fetch (and only called once, right
|
||||||
// after — not polled): this one reaches an external service (Geofabrik
|
// after — not polled): this one reaches an external service (Geofabrik
|
||||||
// or Mapterhorn's manifest, see lib/maptool/pmtiles_builder.rb), so it's
|
// or Mapterhorn's manifest, see lib/maptool/pmtiles_builder.rb), so it's
|
||||||
@@ -249,6 +282,7 @@ export class OfflineDataControl {
|
|||||||
new DataFileRow({
|
new DataFileRow({
|
||||||
label: "Offline map data",
|
label: "Offline map data",
|
||||||
statusUrl: "/api/offline-tiles",
|
statusUrl: "/api/offline-tiles",
|
||||||
|
deleteUrl: "/api/offline-tiles",
|
||||||
buildUrl: "/api/offline-tiles/build",
|
buildUrl: "/api/offline-tiles/build",
|
||||||
buildStatusUrl: "/api/offline-tiles/build-status",
|
buildStatusUrl: "/api/offline-tiles/build-status",
|
||||||
checkUpdateUrl: "/api/offline-tiles/check-update",
|
checkUpdateUrl: "/api/offline-tiles/check-update",
|
||||||
@@ -257,6 +291,7 @@ export class OfflineDataControl {
|
|||||||
new DataFileRow({
|
new DataFileRow({
|
||||||
label: "Offline terrain data",
|
label: "Offline terrain data",
|
||||||
statusUrl: "/api/terrain-tiles",
|
statusUrl: "/api/terrain-tiles",
|
||||||
|
deleteUrl: "/api/terrain-tiles",
|
||||||
buildUrl: "/api/terrain-tiles/build",
|
buildUrl: "/api/terrain-tiles/build",
|
||||||
buildStatusUrl: "/api/terrain-tiles/build-status",
|
buildStatusUrl: "/api/terrain-tiles/build-status",
|
||||||
checkUpdateUrl: "/api/terrain-tiles/check-update",
|
checkUpdateUrl: "/api/terrain-tiles/check-update",
|
||||||
|
|||||||
@@ -72,6 +72,9 @@ class AppTest < Minitest::Test
|
|||||||
assert_equal 401, request.post("/api/plugins/test-plugin/run").status
|
assert_equal 401, request.post("/api/plugins/test-plugin/run").status
|
||||||
assert_equal 401, request.post("/api/offline-tiles/build").status
|
assert_equal 401, request.post("/api/offline-tiles/build").status
|
||||||
assert_equal 401, request.post("/api/terrain-tiles/build").status
|
assert_equal 401, request.post("/api/terrain-tiles/build").status
|
||||||
|
assert_equal 401, request.delete("/api/offline-tiles").status
|
||||||
|
assert_equal 401, request.delete("/api/offline-tiles/cache/SK").status
|
||||||
|
assert_equal 401, request.delete("/api/terrain-tiles").status
|
||||||
end
|
end
|
||||||
|
|
||||||
# --- GPX/FIT upload and the tree ---
|
# --- GPX/FIT upload and the tree ---
|
||||||
@@ -417,6 +420,35 @@ class AppTest < Minitest::Test
|
|||||||
thread&.join
|
thread&.join
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def test_offline_tiles_delete_is_ok_when_nothing_was_built
|
||||||
|
assert_equal 200, request.delete("/api/offline-tiles", **logged_in_env).status
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_offline_tiles_delete_removes_the_file_and_cache
|
||||||
|
path = File.join(Maptool::App::DATA_DIR, "offline.pmtiles")
|
||||||
|
cache_dir = File.join(Maptool::App::DATA_DIR, "geofabrik_cache")
|
||||||
|
File.write(path, "fake")
|
||||||
|
FileUtils.mkdir_p(cache_dir)
|
||||||
|
File.write(File.join(cache_dir, "SK.osm.pbf"), "fake")
|
||||||
|
|
||||||
|
response = request.delete("/api/offline-tiles", **logged_in_env)
|
||||||
|
|
||||||
|
assert_equal 200, response.status
|
||||||
|
refute_path_exists path
|
||||||
|
refute_path_exists cache_dir
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_offline_tiles_delete_is_409_when_a_build_is_running
|
||||||
|
Maptool::PmtilesBuilder.set_selected_regions(Maptool::App::DATA_DIR, ["SK"])
|
||||||
|
gate = Queue.new
|
||||||
|
thread = Maptool::PmtilesBuilder.build_offline(data_dir: Maptool::App::DATA_DIR, download_extract: ->(_url, _dest) { gate.pop })
|
||||||
|
|
||||||
|
assert_equal 409, request.delete("/api/offline-tiles", **logged_in_env).status
|
||||||
|
ensure
|
||||||
|
gate.push("unused")
|
||||||
|
thread&.join
|
||||||
|
end
|
||||||
|
|
||||||
# --- LiveTrack webhook ---
|
# --- LiveTrack webhook ---
|
||||||
|
|
||||||
def test_livetrack_webhook_requires_the_configured_secret
|
def test_livetrack_webhook_requires_the_configured_secret
|
||||||
|
|||||||
@@ -102,8 +102,12 @@ class PmtilesBuilderTest < Minitest::Test
|
|||||||
# download_extract stub: writes a marker file (same "fake" content the
|
# download_extract stub: writes a marker file (same "fake" content the
|
||||||
# pmtiles/osmium/java fixtures use) instead of hitting Geofabrik, and
|
# pmtiles/osmium/java fixtures use) instead of hitting Geofabrik, and
|
||||||
# returns a fixed Last-Modified so sidecar-meta assertions are stable.
|
# returns a fixed Last-Modified so sidecar-meta assertions are stable.
|
||||||
def fake_download(last_modified: "Wed, 10 Sep 2026 08:00:01 GMT")
|
# `calls` (an Array, shared across builds if the caller wants it) records
|
||||||
->(_url, dest) {
|
# every URL actually downloaded, for asserting the per-country cache
|
||||||
|
# skipped (or didn't skip) a re-download.
|
||||||
|
def fake_download(last_modified: "Wed, 10 Sep 2026 08:00:01 GMT", calls: nil)
|
||||||
|
->(url, dest) {
|
||||||
|
calls << url if calls
|
||||||
File.write(dest, "fake")
|
File.write(dest, "fake")
|
||||||
last_modified
|
last_modified
|
||||||
}
|
}
|
||||||
@@ -167,7 +171,7 @@ class PmtilesBuilderTest < Minitest::Test
|
|||||||
thread.join
|
thread.join
|
||||||
|
|
||||||
assert_path_exists File.join(@data_dir, "offline.pmtiles")
|
assert_path_exists File.join(@data_dir, "offline.pmtiles")
|
||||||
refute_path_exists File.join(@data_dir, "offline.pmtiles.tmp")
|
refute_path_exists File.join(@data_dir, "offline.tmp.pmtiles")
|
||||||
assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state]
|
assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state]
|
||||||
assert_equal ["planetiler --osm-path"], log_lines
|
assert_equal ["planetiler --osm-path"], log_lines
|
||||||
assert_equal(
|
assert_equal(
|
||||||
@@ -189,6 +193,151 @@ class PmtilesBuilderTest < Minitest::Test
|
|||||||
assert_equal %w[SK AT], codes
|
assert_equal %w[SK AT], codes
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# The actual bug report this covers: hitting "Apply" after tweaking the
|
||||||
|
# region selection used to re-download every selected country's .osm.pbf
|
||||||
|
# from scratch, even ones already fetched by an earlier build.
|
||||||
|
def test_build_offline_skips_redownloading_an_unchanged_cached_country
|
||||||
|
select_regions("SK")
|
||||||
|
calls = []
|
||||||
|
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
|
||||||
|
assert_equal 1, calls.size
|
||||||
|
|
||||||
|
Maptool::PmtilesBuilder.build_offline(
|
||||||
|
data_dir: @data_dir,
|
||||||
|
download_extract: fake_download(calls: calls),
|
||||||
|
fetch_last_modified: ->(_url) { "Wed, 10 Sep 2026 08:00:01 GMT" },
|
||||||
|
).join
|
||||||
|
|
||||||
|
assert_equal 1, calls.size
|
||||||
|
assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state]
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_build_offline_redownloads_a_country_whose_extract_changed
|
||||||
|
select_regions("SK")
|
||||||
|
calls = []
|
||||||
|
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
|
||||||
|
|
||||||
|
Maptool::PmtilesBuilder.build_offline(
|
||||||
|
data_dir: @data_dir,
|
||||||
|
download_extract: fake_download(calls: calls, last_modified: "Thu, 11 Sep 2026 08:00:01 GMT"),
|
||||||
|
fetch_last_modified: ->(_url) { "Thu, 11 Sep 2026 08:00:01 GMT" },
|
||||||
|
).join
|
||||||
|
|
||||||
|
assert_equal 2, calls.size
|
||||||
|
end
|
||||||
|
|
||||||
|
# A cache-freshness check that itself fails (Geofabrik unreachable) should
|
||||||
|
# fall back to re-downloading rather than aborting the build.
|
||||||
|
def test_build_offline_redownloads_when_the_cache_check_fails
|
||||||
|
select_regions("SK")
|
||||||
|
calls = []
|
||||||
|
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
|
||||||
|
|
||||||
|
fetch_last_modified = ->(_url) { raise Maptool::PmtilesBuilder::BuildError, "could not reach geofabrik" }
|
||||||
|
Maptool::PmtilesBuilder.build_offline(
|
||||||
|
data_dir: @data_dir,
|
||||||
|
download_extract: fake_download(calls: calls),
|
||||||
|
fetch_last_modified: fetch_last_modified,
|
||||||
|
).join
|
||||||
|
|
||||||
|
assert_equal 2, calls.size
|
||||||
|
assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state]
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_build_offline_only_downloads_a_newly_added_country
|
||||||
|
select_regions("SK")
|
||||||
|
calls = []
|
||||||
|
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
|
||||||
|
|
||||||
|
select_regions("SK", "AT")
|
||||||
|
Maptool::PmtilesBuilder.build_offline(
|
||||||
|
data_dir: @data_dir,
|
||||||
|
download_extract: fake_download(calls: calls),
|
||||||
|
fetch_last_modified: ->(_url) { "Wed, 10 Sep 2026 08:00:01 GMT" },
|
||||||
|
).join
|
||||||
|
|
||||||
|
assert_equal 1, calls.count { |url| url.include?("slovakia") }
|
||||||
|
assert_equal 1, calls.count { |url| url.include?("austria") }
|
||||||
|
end
|
||||||
|
|
||||||
|
# offline_status[:step] should say which country is in flight right now
|
||||||
|
# and which of the current build's selection are already done/still
|
||||||
|
# pending — not just "running since X".
|
||||||
|
def test_build_offline_reports_progress_step_while_running
|
||||||
|
select_regions("SK", "AT")
|
||||||
|
started = Queue.new
|
||||||
|
gate = Queue.new
|
||||||
|
download_extract = ->(url, dest) {
|
||||||
|
started.push(url)
|
||||||
|
gate.pop
|
||||||
|
File.write(dest, "fake")
|
||||||
|
"Wed, 10 Sep 2026 08:00:01 GMT"
|
||||||
|
}
|
||||||
|
thread = Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: download_extract)
|
||||||
|
|
||||||
|
started.pop
|
||||||
|
assert_equal "fetching SK (1/2) — pending: AT", Maptool::PmtilesBuilder.offline_status[:step]
|
||||||
|
gate.push(:go)
|
||||||
|
|
||||||
|
started.pop
|
||||||
|
assert_equal "fetching AT (2/2) — done: SK", Maptool::PmtilesBuilder.offline_status[:step]
|
||||||
|
gate.push(:go)
|
||||||
|
|
||||||
|
thread.join
|
||||||
|
assert_nil Maptool::PmtilesBuilder.offline_status[:step]
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_delete_offline_data_removes_the_file_meta_and_cache
|
||||||
|
select_regions("SK")
|
||||||
|
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download).join
|
||||||
|
assert_path_exists File.join(@data_dir, "offline.pmtiles")
|
||||||
|
assert_path_exists File.join(@data_dir, "geofabrik_cache", "SK.osm.pbf")
|
||||||
|
|
||||||
|
Maptool::PmtilesBuilder.delete_offline_data(@data_dir)
|
||||||
|
|
||||||
|
refute_path_exists File.join(@data_dir, "offline.pmtiles")
|
||||||
|
refute_path_exists File.join(@data_dir, "geofabrik_cache")
|
||||||
|
assert_nil Maptool::PmtilesBuilder.offline_meta(@data_dir)
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_delete_offline_data_is_a_noop_when_nothing_was_built
|
||||||
|
Maptool::PmtilesBuilder.delete_offline_data(@data_dir) # should not raise
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_delete_offline_data_raises_while_a_build_is_running
|
||||||
|
select_regions("SK")
|
||||||
|
gate = Queue.new
|
||||||
|
thread = Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: ->(_url, _dest) { gate.pop })
|
||||||
|
|
||||||
|
assert_raises(Maptool::PmtilesBuilder::BuildError) { Maptool::PmtilesBuilder.delete_offline_data(@data_dir) }
|
||||||
|
ensure
|
||||||
|
gate.push("unused")
|
||||||
|
thread&.join
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_delete_cached_extract_removes_only_that_country
|
||||||
|
select_regions("SK", "AT")
|
||||||
|
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download).join
|
||||||
|
|
||||||
|
Maptool::PmtilesBuilder.delete_cached_extract(@data_dir, "SK")
|
||||||
|
|
||||||
|
refute_path_exists File.join(@data_dir, "geofabrik_cache", "SK.osm.pbf")
|
||||||
|
assert_path_exists File.join(@data_dir, "geofabrik_cache", "AT.osm.pbf")
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_delete_terrain_data_removes_the_file_and_meta
|
||||||
|
select_regions("ES")
|
||||||
|
ENV["PMTILES_BIN"] = @pmtiles_ok
|
||||||
|
thread = Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
|
||||||
|
thread.join
|
||||||
|
assert_path_exists File.join(@data_dir, "terrain.pmtiles")
|
||||||
|
|
||||||
|
Maptool::PmtilesBuilder.delete_terrain_data(@data_dir)
|
||||||
|
|
||||||
|
refute_path_exists File.join(@data_dir, "terrain.pmtiles")
|
||||||
|
assert_nil Maptool::PmtilesBuilder.terrain_meta(@data_dir)
|
||||||
|
end
|
||||||
|
|
||||||
def test_offline_meta_is_nil_before_any_build
|
def test_offline_meta_is_nil_before_any_build
|
||||||
assert_nil Maptool::PmtilesBuilder.offline_meta(@data_dir)
|
assert_nil Maptool::PmtilesBuilder.offline_meta(@data_dir)
|
||||||
end
|
end
|
||||||
|
|||||||
Reference in new issue
Block a user