Fix Planetiler tmp-file format detection, add build progress and delete
Root cause of "planetiler exited with status 1: ... Unsupported format
tmp" (and likely the earlier unexplained "pmtiles exited with status
1"): Planetiler/go-pmtiles pick their output archive format from the
file extension, and the atomic-rename tmp files were named
"offline.pmtiles.tmp"/"terrain.pmtiles.tmp" — ".tmp" as the extension,
not ".pmtiles". Renamed to "offline.tmp.pmtiles"/"terrain.tmp.pmtiles".
Also, since build-status only ever said "running since X" with no way
to tell what it was actually doing:
- GET .../build-status now includes a `step` string ("fetching PL
(3/4) — done: SK, CZ — pending: HU", "tiling with planetiler", ...)
while a build is in progress.
- DELETE /api/offline-tiles / /api/terrain-tiles clear a built file,
its sidecar meta, and (for offline) the whole per-country download
cache — a manual "reclaim space" / "start over" action the region
picker's Apply never did on its own. DELETE
/api/offline-tiles/cache/:code clears just one country's cached
extract. All three are 409 while that kind's build is running, and
surfaced as a small "Delete" button on each row in the Offline data
panel.
This commit is contained in:
1 parent
2878270187
commit
b54020e23b
7 files changed
+327
-23
No files matched your search
@@ -189,7 +189,11 @@ module Maptool
|
||||
raise BuildError, "select at least one region first" if codes.empty?
|
||||
|
||||
begin_run!(OFFLINE_MUTEX, OFFLINE_STATUS)
|
||||
Thread.new { run_job(OFFLINE_MUTEX, OFFLINE_STATUS) { extract_offline(codes, data_dir, download_extract, fetch_last_modified) } }
|
||||
Thread.new do
|
||||
run_job(OFFLINE_MUTEX, OFFLINE_STATUS) do |report_step|
|
||||
extract_offline(codes, data_dir, download_extract, fetch_last_modified, report_step)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# fetch_manifest: same injectable-seam reasoning as download_extract
|
||||
@@ -199,7 +203,11 @@ module Maptool
|
||||
raise BuildError, "select at least one region first" if codes.empty?
|
||||
|
||||
begin_run!(TERRAIN_MUTEX, TERRAIN_STATUS)
|
||||
Thread.new { run_job(TERRAIN_MUTEX, TERRAIN_STATUS) { build_terrain_pmtiles(codes, data_dir, fetch_manifest) } }
|
||||
Thread.new do
|
||||
run_job(TERRAIN_MUTEX, TERRAIN_STATUS) do |report_step|
|
||||
build_terrain_pmtiles(codes, data_dir, fetch_manifest, report_step)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def self.begin_run!(mutex, status)
|
||||
@@ -210,11 +218,19 @@ module Maptool
|
||||
end
|
||||
private_class_method :begin_run!
|
||||
|
||||
# Yields a report_step proc the build can call with a short
|
||||
# human-readable description of what it's doing right now ("downloading
|
||||
# SK (1/2)", "tiling with planetiler", ...) — surfaced as status[:step]
|
||||
# so GET .../build-status can show progress beyond just "running since
|
||||
# X" (see docs/development.adoc).
|
||||
def self.run_job(mutex, status)
|
||||
yield
|
||||
report_step = ->(text) { mutex.synchronize { status[:step] = text } }
|
||||
yield report_step
|
||||
mutex.synchronize { status.merge!(state: "done", finished_at: Time.now.utc.iso8601).delete(:message) }
|
||||
rescue => e
|
||||
mutex.synchronize { status.merge!(state: "error", finished_at: Time.now.utc.iso8601, message: e.message) }
|
||||
ensure
|
||||
mutex.synchronize { status.delete(:step) }
|
||||
end
|
||||
private_class_method :run_job
|
||||
|
||||
@@ -228,10 +244,11 @@ module Maptool
|
||||
# tiles.openfreemap.org's Liberty style already expects, so
|
||||
# public/js/map.js's offlineStyle() can reuse ONLINE_STYLE_PATCH
|
||||
# unchanged instead of a separate protomaps-themes-base palette.
|
||||
def self.extract_offline(codes, data_dir, download_extract, fetch_last_modified)
|
||||
def self.extract_offline(codes, data_dir, download_extract, fetch_last_modified, report_step)
|
||||
scratch_dir = Dir.mktmpdir("maptool-offline")
|
||||
sources = codes.map do |code|
|
||||
sources = codes.each_with_index.map do |code, i|
|
||||
slug = GEOFABRIK_SLUGS.fetch(code) { raise BuildError, "no Geofabrik source known for region #{code}" }
|
||||
report_step.call(progress_step("fetching #{code}", codes, i))
|
||||
cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
|
||||
end
|
||||
|
||||
@@ -239,12 +256,18 @@ module Maptool
|
||||
if sources.size == 1
|
||||
sources.first["path"]
|
||||
else
|
||||
report_step.call("merging with osmium")
|
||||
merged_path = File.join(scratch_dir, "merged.osm.pbf")
|
||||
run_osmium!("merge", *sources.map { |s| s["path"] }, "-o", merged_path)
|
||||
merged_path
|
||||
end
|
||||
|
||||
tmp_path = File.join(data_dir, "offline.pmtiles.tmp")
|
||||
report_step.call("tiling with planetiler")
|
||||
# .tmp.pmtiles, not .pmtiles.tmp — Planetiler picks its output archive
|
||||
# format (pmtiles vs mbtiles) from the file extension, so a ".tmp"
|
||||
# suffix made it fail with "Unsupported format tmp" instead of
|
||||
# actually tiling anything.
|
||||
tmp_path = File.join(data_dir, "offline.tmp.pmtiles")
|
||||
run_planetiler!("--osm-path=#{osm_path}", "--output=#{tmp_path}")
|
||||
File.rename(tmp_path, File.join(data_dir, "offline.pmtiles"))
|
||||
File.write(offline_meta_path(data_dir), JSON.generate(sources: sources.map { |s| s.slice("code", "slug", "last_modified") }))
|
||||
@@ -254,6 +277,20 @@ module Maptool
|
||||
end
|
||||
private_class_method :extract_offline
|
||||
|
||||
# Renders a report_step string like "fetching PL (3/4) — done: SK, CZ —
|
||||
# pending: HU" — so GET .../build-status shows not just "it's running"
|
||||
# but which of the current build's sources is in flight and which are
|
||||
# already behind/ahead of it.
|
||||
def self.progress_step(action, labels, index)
|
||||
done = labels[0...index]
|
||||
pending = labels[(index + 1)..] || []
|
||||
parts = ["#{action} (#{index + 1}/#{labels.size})"]
|
||||
parts << "done: #{done.join(", ")}" unless done.empty?
|
||||
parts << "pending: #{pending.join(", ")}" unless pending.empty?
|
||||
parts.join(" — ")
|
||||
end
|
||||
private_class_method :progress_step
|
||||
|
||||
def self.geofabrik_url(slug)
|
||||
"https://download.geofabrik.de/#{slug}-latest.osm.pbf"
|
||||
end
|
||||
@@ -378,6 +415,31 @@ module Maptool
|
||||
end
|
||||
private_class_method :offline_meta_path
|
||||
|
||||
# Deletes offline.pmtiles, its sidecar meta, and the whole per-country
|
||||
# download cache (data/geofabrik_cache/) — "start over" for the offline
|
||||
# basemap. The current selection (data/offline_regions.json) is left
|
||||
# alone, so a following build just re-downloads it fresh.
|
||||
def self.delete_offline_data(data_dir)
|
||||
OFFLINE_MUTEX.synchronize { raise BuildError, "a build is already running" if OFFLINE_STATUS[:state] == "running" }
|
||||
|
||||
File.delete(File.join(data_dir, "offline.pmtiles")) if File.exist?(File.join(data_dir, "offline.pmtiles"))
|
||||
File.delete(offline_meta_path(data_dir)) if File.exist?(offline_meta_path(data_dir))
|
||||
cache_dir = geofabrik_cache_dir(data_dir)
|
||||
FileUtils.remove_entry(cache_dir) if Dir.exist?(cache_dir)
|
||||
end
|
||||
|
||||
# Deletes just one country's cached .osm.pbf (and its Last-Modified
|
||||
# stamp) — e.g. to force a fresh download without touching the others,
|
||||
# or to reclaim space for a country no longer selected.
|
||||
def self.delete_cached_extract(data_dir, code)
|
||||
OFFLINE_MUTEX.synchronize { raise BuildError, "a build is already running" if OFFLINE_STATUS[:state] == "running" }
|
||||
|
||||
cache_dir = geofabrik_cache_dir(data_dir)
|
||||
path = File.join(cache_dir, "#{code}.osm.pbf")
|
||||
File.delete(path) if File.exist?(path)
|
||||
File.delete("#{path}.last_modified") if File.exist?("#{path}.last_modified")
|
||||
end
|
||||
|
||||
# --- terrain.pmtiles ---
|
||||
|
||||
# planet.pmtiles (z0-12, always) plus whichever regional chunks
|
||||
@@ -394,7 +456,8 @@ module Maptool
|
||||
# border tiles (extraction copies whole tiles, it doesn't clip vector
|
||||
# content to the polygon) — so this always does one extract per
|
||||
# *source archive*, never one per country.
|
||||
def self.build_terrain_pmtiles(codes, data_dir, fetch_manifest)
|
||||
def self.build_terrain_pmtiles(codes, data_dir, fetch_manifest, report_step)
|
||||
report_step.call("fetching mapterhorn manifest")
|
||||
items = fetch_manifest.call
|
||||
planet = items.find { |item| item["name"] == MAPTERHORN_PLANET_NAME }
|
||||
raise BuildError, "mapterhorn manifest is missing #{MAPTERHORN_PLANET_NAME}" unless planet
|
||||
@@ -407,8 +470,14 @@ module Maptool
|
||||
|
||||
chunks = items.select { |item| item["name"] != MAPTERHORN_PLANET_NAME && bbox_intersects?(item, south, west, north, east) }
|
||||
|
||||
tmp_path = File.join(data_dir, "terrain.pmtiles.tmp")
|
||||
extracts = [planet, *chunks].each_with_index.map do |item, i|
|
||||
# .tmp.pmtiles, not .pmtiles.tmp — same reasoning as offline.pmtiles'
|
||||
# own tmp_path above, in case go-pmtiles also derives format from
|
||||
# the extension for its "merge" output.
|
||||
tmp_path = File.join(data_dir, "terrain.tmp.pmtiles")
|
||||
sources = [planet, *chunks]
|
||||
labels = sources.map { |item| item["name"] }
|
||||
extracts = sources.each_with_index.map do |item, i|
|
||||
report_step.call(progress_step("extracting #{item["name"]}", labels, i))
|
||||
extract_path = File.join(scratch_dir, "extract-#{i}.pmtiles")
|
||||
run_pmtiles!("extract", item["url"], extract_path, "--region=#{region_path}", "--maxzoom=#{TERRAIN_MAX_ZOOM}")
|
||||
extract_path
|
||||
@@ -417,6 +486,7 @@ module Maptool
|
||||
if extracts.size == 1
|
||||
File.rename(extracts.first, tmp_path)
|
||||
else
|
||||
report_step.call("merging with pmtiles")
|
||||
run_pmtiles!("merge", *extracts, tmp_path)
|
||||
end
|
||||
File.rename(tmp_path, File.join(data_dir, "terrain.pmtiles"))
|
||||
@@ -479,6 +549,17 @@ module Maptool
|
||||
end
|
||||
private_class_method :terrain_meta_path
|
||||
|
||||
# Deletes terrain.pmtiles and its sidecar meta — same "start over"
|
||||
# reasoning as delete_offline_data above. No download cache to clean up
|
||||
# here: terrain.pmtiles' pmtiles-extract step reads its sources over
|
||||
# HTTP range requests rather than downloading a whole archive first.
|
||||
def self.delete_terrain_data(data_dir)
|
||||
TERRAIN_MUTEX.synchronize { raise BuildError, "a build is already running" if TERRAIN_STATUS[:state] == "running" }
|
||||
|
||||
File.delete(File.join(data_dir, "terrain.pmtiles")) if File.exist?(File.join(data_dir, "terrain.pmtiles"))
|
||||
File.delete(terrain_meta_path(data_dir)) if File.exist?(terrain_meta_path(data_dir))
|
||||
end
|
||||
|
||||
# --- shared ---
|
||||
|
||||
def self.read_json(path)
|
||||
|
||||
Reference in new issue
Block a user