664 lines
30 KiB
Ruby
664 lines
30 KiB
Ruby
require "open3"
|
|
require "net/http"
|
|
require "uri"
|
|
require "json"
|
|
require "tmpdir"
|
|
require "fileutils"
|
|
require "etc"
|
|
|
|
module Maptool
|
|
# Regenerates data/offline.pmtiles and data/terrain.pmtiles for the
|
|
# currently *selected* regions (data/offline_regions.json, an array of
|
|
# Natural Earth iso_a2 country codes — see available_regions/
|
|
# selected_regions/set_selected_regions below). No job queue: each build
|
|
# runs on its own background Thread and reports into a Mutex-guarded
|
|
# status Hash a route can poll, the same shape app.rb's own GNSS poller
|
|
# (GNSS_LATEST/GNSS_LATEST_MUTEX) already uses.
|
|
module PmtilesBuilder
|
|
class BuildError < StandardError; end
|
|
|
|
# Natural Earth (public domain, no attribution required) 110m admin-0
|
|
# countries, filtered to Europe and trimmed to name/iso_a2/iso_a3 —
|
|
# see the plan this was built from for the exact filter. 110m is
|
|
# deliberately coarse (a km or so of border precision, not
|
|
# survey-grade) — fine for "which country to grab," a same-schema swap
|
|
# to Natural Earth's 50m/10m cut if that ever needs to be tighter.
|
|
REGIONS_FILE = File.join(__dir__, "data", "europe_countries.geojson")
|
|
|
|
# Every iso_a2 code in REGIONS_FILE mapped to its Geofabrik download
|
|
# slug (https://download.geofabrik.de/<slug>-latest.osm.pbf) — Geofabrik
|
|
# names regions, not ISO codes, and doesn't always split the way a
|
|
# country border does, so this table exists rather than deriving the
|
|
# slug from the country name. Two known quirks worth knowing about,
|
|
# not worth working around:
|
|
# - RU has no "europe/" prefix (Russia is its own top-level index,
|
|
# and unlike every other entry here is a single ~4GB file covering
|
|
# the whole country rather than a European-only slice).
|
|
# - GB's "great-britain" slug is England/Scotland/Wales only —
|
|
# Northern Ireland lives in "ireland-and-northern-ireland" instead
|
|
# (bundled with IE, not GB). Selecting GB alone won't include NI.
|
|
GEOFABRIK_SLUGS = {
|
|
"AL" => "europe/albania",
|
|
"AT" => "europe/austria",
|
|
"BY" => "europe/belarus",
|
|
"BE" => "europe/belgium",
|
|
"BA" => "europe/bosnia-herzegovina",
|
|
"BG" => "europe/bulgaria",
|
|
"HR" => "europe/croatia",
|
|
"CZ" => "europe/czech-republic",
|
|
"DK" => "europe/denmark",
|
|
"EE" => "europe/estonia",
|
|
"FI" => "europe/finland",
|
|
"FR" => "europe/france",
|
|
"DE" => "europe/germany",
|
|
"GR" => "europe/greece",
|
|
"HU" => "europe/hungary",
|
|
"IS" => "europe/iceland",
|
|
"IE" => "europe/ireland-and-northern-ireland",
|
|
"IT" => "europe/italy",
|
|
"XK" => "europe/kosovo",
|
|
"LV" => "europe/latvia",
|
|
"LT" => "europe/lithuania",
|
|
"LU" => "europe/luxembourg",
|
|
"MD" => "europe/moldova",
|
|
"ME" => "europe/montenegro",
|
|
"NL" => "europe/netherlands",
|
|
"MK" => "europe/macedonia",
|
|
"NO" => "europe/norway",
|
|
"PL" => "europe/poland",
|
|
"PT" => "europe/portugal",
|
|
"RO" => "europe/romania",
|
|
"RU" => "russia",
|
|
"RS" => "europe/serbia",
|
|
"SK" => "europe/slovakia",
|
|
"SI" => "europe/slovenia",
|
|
"ES" => "europe/spain",
|
|
"SE" => "europe/sweden",
|
|
"CH" => "europe/switzerland",
|
|
"UA" => "europe/ukraine",
|
|
"GB" => "europe/great-britain",
|
|
}.freeze
|
|
|
|
# Mapterhorn (a Protomaps-affiliated project) publishes Terrarium-
|
|
# encoded terrain as pre-built PMTiles archives: one global
|
|
# planet.pmtiles (z0-12) plus ~450 regional chunks (z13 up to a
|
|
# per-region max, 13-18) — extract_offline below no longer touches
|
|
# Protomaps at all (see GEOFABRIK_SLUGS above), but terrain stays on
|
|
# this same "extract only the region's byte ranges from a remote
|
|
# archive" story via go-pmtiles, unrelated to the basemap swap.
|
|
MAPTERHORN_MANIFEST_URL = "https://download.mapterhorn.com/download_urls.json"
|
|
MAPTERHORN_PLANET_NAME = "planet.pmtiles"
|
|
# Matches the online hillshade layer's own cap (public/js/overlays.js's
|
|
# ONLINE_TERRAIN_TILES source, maxzoom: 15) — applied uniformly even
|
|
# though some Mapterhorn chunks natively offer more, so the offline
|
|
# experience doesn't vary in detail from one region to the next.
|
|
TERRAIN_MAX_ZOOM = 15
|
|
|
|
OFFLINE_STATUS = { state: "idle" }
|
|
TERRAIN_STATUS = { state: "idle" }
|
|
OFFLINE_MUTEX = Mutex.new
|
|
TERRAIN_MUTEX = Mutex.new
|
|
|
|
def self.offline_status
|
|
OFFLINE_MUTEX.synchronize { OFFLINE_STATUS.dup }
|
|
end
|
|
|
|
def self.terrain_status
|
|
TERRAIN_MUTEX.synchronize { TERRAIN_STATUS.dup }
|
|
end
|
|
|
|
# Test-only: the status hashes are process-wide state (same caveat as
|
|
# GPX_STORE/LIVETRACK_STORE in app.rb), so the suite needs a way to
|
|
# reset them between tests the way AppTestHelper#teardown already does
|
|
# for those.
|
|
def self.reset!
|
|
OFFLINE_MUTEX.synchronize { OFFLINE_STATUS.replace(state: "idle") }
|
|
TERRAIN_MUTEX.synchronize { TERRAIN_STATUS.replace(state: "idle") }
|
|
end
|
|
|
|
# --- region selection ---
|
|
|
|
def self.available_regions
|
|
region_features.map { |f| { code: f["properties"]["iso_a2"], name: f["properties"]["name"] } }.sort_by { |r| r[:name] }
|
|
end
|
|
|
|
def self.selected_regions(data_dir)
|
|
path = selected_regions_path(data_dir)
|
|
return [] unless File.exist?(path)
|
|
|
|
JSON.parse(File.read(path))
|
|
rescue JSON::ParserError
|
|
[]
|
|
end
|
|
|
|
def self.set_selected_regions(data_dir, codes)
|
|
unknown = codes - available_regions.map { |r| r[:code] }
|
|
raise BuildError, "unknown region code(s): #{unknown.join(", ")}" unless unknown.empty?
|
|
|
|
File.write(selected_regions_path(data_dir), JSON.generate(codes))
|
|
end
|
|
|
|
def self.selected_regions_path(data_dir)
|
|
File.join(data_dir, "offline_regions.json")
|
|
end
|
|
private_class_method :selected_regions_path
|
|
|
|
def self.region_features
|
|
@region_features ||= JSON.parse(File.read(REGIONS_FILE))["features"]
|
|
end
|
|
private_class_method :region_features
|
|
|
|
# Still used by terrain (below) to compute the aggregate bbox Mapterhorn
|
|
# chunk intersection needs — offline.pmtiles no longer needs this at
|
|
# all, since each Geofabrik country file is already clipped to that
|
|
# country's real border (more accurate than this 110m polygon, not
|
|
# less), so there's no separate "--region=" precision step for it
|
|
# anymore.
|
|
def self.region_bbox(codes)
|
|
selected = region_features.select { |f| codes.include?(f["properties"]["iso_a2"]) }
|
|
raise BuildError, "no matching regions for #{codes.join(", ")}" if selected.empty?
|
|
|
|
coords = selected.flat_map { |f| ring_coordinates(f["geometry"]) }
|
|
lons = coords.map { |lng, _lat| lng }
|
|
lats = coords.map { |_lng, lat| lat }
|
|
[lats.min, lons.min, lats.max, lons.max]
|
|
end
|
|
private_class_method :region_bbox
|
|
|
|
def self.ring_coordinates(geometry)
|
|
case geometry["type"]
|
|
when "Polygon" then geometry["coordinates"].flatten(1)
|
|
when "MultiPolygon" then geometry["coordinates"].flatten(2)
|
|
else []
|
|
end
|
|
end
|
|
private_class_method :ring_coordinates
|
|
|
|
# --- builds ---
|
|
|
|
# Returns once the background Thread has started, not once the build
|
|
# finishes — raises BuildError (the caller turns this into an error
|
|
# response) if nothing's selected yet, or one's already running.
|
|
#
|
|
# download_extract/fetch_last_modified exist as injectable seams so
|
|
# tests can exercise the rest of this (scratch dir, osmium/planetiler
|
|
# invocation, atomic rename, the per-country cache below) without a
|
|
# real network call — same reasoning test_helper.rb already gives for
|
|
# keeping /api/plugins route tests off the real Overpass API.
|
|
def self.build_offline(data_dir:, download_extract: method(:download_extract), fetch_last_modified: method(:fetch_geofabrik_last_modified))
|
|
codes = selected_regions(data_dir)
|
|
raise BuildError, "select at least one region first" if codes.empty?
|
|
|
|
begin_run!(OFFLINE_MUTEX, OFFLINE_STATUS)
|
|
Thread.new do
|
|
run_job(OFFLINE_MUTEX, OFFLINE_STATUS) do |report_step|
|
|
extract_offline(codes, data_dir, download_extract, fetch_last_modified, report_step)
|
|
end
|
|
end
|
|
end
|
|
|
|
# fetch_manifest: same injectable-seam reasoning as download_extract
|
|
# above, for the one Mapterhorn manifest lookup this needs.
|
|
def self.build_terrain(data_dir:, fetch_manifest: method(:fetch_mapterhorn_manifest))
|
|
codes = selected_regions(data_dir)
|
|
raise BuildError, "select at least one region first" if codes.empty?
|
|
|
|
begin_run!(TERRAIN_MUTEX, TERRAIN_STATUS)
|
|
Thread.new do
|
|
run_job(TERRAIN_MUTEX, TERRAIN_STATUS) do |report_step|
|
|
build_terrain_pmtiles(codes, data_dir, fetch_manifest, report_step)
|
|
end
|
|
end
|
|
end
|
|
|
|
def self.begin_run!(mutex, status)
|
|
mutex.synchronize do
|
|
raise BuildError, "a build is already running" if status[:state] == "running"
|
|
status.replace(state: "running", started_at: Time.now.utc.iso8601)
|
|
end
|
|
end
|
|
private_class_method :begin_run!
|
|
|
|
# Yields a report_step proc the build can call with a short
|
|
# human-readable description of what it's doing right now ("downloading
|
|
# SK (1/2)", "tiling with planetiler", ...) — surfaced as status[:step]
|
|
# so GET .../build-status can show progress beyond just "running since
|
|
# X" (see docs/development.adoc).
|
|
def self.run_job(mutex, status)
|
|
report_step = ->(text) { mutex.synchronize { status[:step] = text } }
|
|
yield report_step
|
|
mutex.synchronize { status.merge!(state: "done", finished_at: Time.now.utc.iso8601).delete(:message) }
|
|
rescue => e
|
|
mutex.synchronize { status.merge!(state: "error", finished_at: Time.now.utc.iso8601, message: e.message) }
|
|
ensure
|
|
mutex.synchronize { status.delete(:step) }
|
|
end
|
|
private_class_method :run_job
|
|
|
|
# --- offline.pmtiles ---
|
|
|
|
# One Geofabrik .osm.pbf per selected country (already clipped to that
|
|
# country's real border), osmium-merged into one file if more than one
|
|
# was selected (skipped entirely for a single country — nothing to
|
|
# merge), then tiled by Planetiler using its default OpenMapTiles
|
|
# profile straight to PMTiles — the same vector schema
|
|
# tiles.openfreemap.org's Liberty style already expects, so
|
|
# public/js/map.js's offlineStyle() can reuse ONLINE_STYLE_PATCH
|
|
# unchanged instead of a separate protomaps-themes-base palette.
|
|
def self.extract_offline(codes, data_dir, download_extract, fetch_last_modified, report_step)
|
|
scratch_dir = Dir.mktmpdir("maptool-offline")
|
|
sources = codes.each_with_index.map do |code, i|
|
|
slug = GEOFABRIK_SLUGS.fetch(code) { raise BuildError, "no Geofabrik source known for region #{code}" }
|
|
report_step.call(progress_step("fetching #{code}", codes, i))
|
|
cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
|
|
end
|
|
|
|
osm_path =
|
|
if sources.size == 1
|
|
sources.first["path"]
|
|
else
|
|
report_step.call("merging with osmium")
|
|
merged_path = File.join(scratch_dir, "merged.osm.pbf")
|
|
run_osmium!("merge", *sources.map { |s| s["path"] }, "-o", merged_path)
|
|
merged_path
|
|
end
|
|
|
|
report_step.call("tiling with planetiler")
|
|
# .tmp.pmtiles, not .pmtiles.tmp — Planetiler picks its output archive
|
|
# format (pmtiles vs mbtiles) from the file extension, so a ".tmp"
|
|
# suffix made it fail with "Unsupported format tmp" instead of
|
|
# actually tiling anything.
|
|
tmp_path = File.join(data_dir, "offline.tmp.pmtiles")
|
|
# --download: the OpenMapTiles profile also needs a few small auxiliary
|
|
# sources (water polygons, Natural Earth, lake centerlines) alongside
|
|
# the OSM extract itself — without this flag Planetiler expects them
|
|
# to already exist on disk and fails instead of fetching them. They
|
|
# land under Planetiler's own default "data/sources/", relative to
|
|
# this process's working directory (the app root, so in practice
|
|
# that's data_dir's own "sources" subdirectory) — fetched once,
|
|
# reused by every later build.
|
|
# --threads: Planetiler's own core-count autodetection can come up
|
|
# short in a container (cgroup/JVM detection quirks vary by runtime),
|
|
# leaving most of a many-core host idle — Etc.nprocessors asks the
|
|
# OS directly (sched_getaffinity on Linux), which correctly reflects
|
|
# whatever's actually usable by this process even under a container.
|
|
run_planetiler!("--osm-path=#{osm_path}", "--output=#{tmp_path}", "--download", "--threads=#{Etc.nprocessors}")
|
|
File.rename(tmp_path, File.join(data_dir, "offline.pmtiles"))
|
|
File.write(offline_meta_path(data_dir), JSON.generate(sources: sources.map { |s| s.slice("code", "slug", "last_modified") }))
|
|
ensure
|
|
FileUtils.remove_entry(scratch_dir) if scratch_dir && Dir.exist?(scratch_dir)
|
|
File.delete(tmp_path) if tmp_path && File.exist?(tmp_path)
|
|
end
|
|
private_class_method :extract_offline
|
|
|
|
# Renders a report_step string like "fetching PL (3/4) — done: SK, CZ —
|
|
# pending: HU" — so GET .../build-status shows not just "it's running"
|
|
# but which of the current build's sources is in flight and which are
|
|
# already behind/ahead of it.
|
|
def self.progress_step(action, labels, index)
|
|
done = labels[0...index]
|
|
pending = labels[(index + 1)..] || []
|
|
parts = ["#{action} (#{index + 1}/#{labels.size})"]
|
|
parts << "done: #{done.join(", ")}" unless done.empty?
|
|
parts << "pending: #{pending.join(", ")}" unless pending.empty?
|
|
parts.join(" — ")
|
|
end
|
|
private_class_method :progress_step
|
|
|
|
def self.geofabrik_url(slug)
|
|
"https://download.geofabrik.de/#{slug}-latest.osm.pbf"
|
|
end
|
|
private_class_method :geofabrik_url
|
|
|
|
# Per-country .osm.pbf cache, kept across builds under data_dir (not
|
|
# scratch_dir, which is wiped every run) — changing the region
|
|
# selection re-downloads only the countries that are new to it, and
|
|
# re-running the same selection re-downloads nothing unless Geofabrik's
|
|
# own Last-Modified for that country moved on. A stamp file next to
|
|
# each cached extract records the Last-Modified it was fetched with;
|
|
# a HEAD check (cheap) decides whether the multi-GB GET (not cheap) is
|
|
# skippable.
|
|
def self.cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
|
|
cache_dir = geofabrik_cache_dir(data_dir)
|
|
FileUtils.mkdir_p(cache_dir)
|
|
path = File.join(cache_dir, "#{code}.osm.pbf")
|
|
stamp_path = "#{path}.last_modified"
|
|
url = geofabrik_url(slug)
|
|
|
|
if File.exist?(path) && File.exist?(stamp_path)
|
|
current = begin
|
|
fetch_last_modified.call(url)
|
|
rescue BuildError
|
|
nil
|
|
end
|
|
return { "code" => code, "slug" => slug, "last_modified" => current, "path" => path } if current && current == File.read(stamp_path)
|
|
end
|
|
|
|
last_modified = download_extract.call(url, path)
|
|
File.write(stamp_path, last_modified.to_s)
|
|
{ "code" => code, "slug" => slug, "last_modified" => last_modified, "path" => path }
|
|
end
|
|
private_class_method :cached_extract
|
|
|
|
def self.geofabrik_cache_dir(data_dir)
|
|
File.join(data_dir, "geofabrik_cache")
|
|
end
|
|
private_class_method :geofabrik_cache_dir
|
|
|
|
# Real network implementation of the download_extract seam above —
|
|
# streams the extract to dest (some of these are multi-GB — Net::HTTP's
|
|
# plain #get buffers the whole response body in memory first, which
|
|
# read_body's block form avoids) and returns its Last-Modified header
|
|
# (free off the same GET, no separate HEAD needed) for the sidecar
|
|
# meta / update-check below. Geofabrik 302s (e.g. .../europe/austria
|
|
# redirecting to wherever actually serves it) — followed manually
|
|
# since request_get's block form doesn't do this itself.
|
|
def self.download_extract(url, dest, limit: 5)
|
|
raise BuildError, "too many redirects downloading #{url}" if limit.zero?
|
|
|
|
uri = URI(url)
|
|
last_modified = nil
|
|
redirect_to = nil
|
|
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
|
|
http.request_get(uri) do |response|
|
|
case response
|
|
when Net::HTTPSuccess
|
|
last_modified = response["last-modified"]
|
|
File.open(dest, "wb") { |file| response.read_body { |chunk| file.write(chunk) } }
|
|
when Net::HTTPRedirection
|
|
redirect_to = URI.join(url, response["location"]).to_s
|
|
else
|
|
raise BuildError, "could not download #{url} (HTTP #{response.code})"
|
|
end
|
|
end
|
|
end
|
|
return download_extract(redirect_to, dest, limit: limit - 1) if redirect_to
|
|
|
|
last_modified
|
|
end
|
|
|
|
# Sidecar recording which Geofabrik sources (country code, slug, and
|
|
# that request's Last-Modified) the on-disk offline.pmtiles was
|
|
# actually built from — the file itself carries no such record. nil if
|
|
# never built (or built before this existed).
|
|
def self.offline_meta(data_dir)
|
|
read_json(offline_meta_path(data_dir))
|
|
end
|
|
|
|
# Compares each currently-selected country's live Geofabrik
|
|
# Last-Modified against what's recorded, without downloading anything
|
|
# — nil (not true/false) if there's nothing to compare yet (no local
|
|
# build, nothing selected) or a check couldn't be completed, so the
|
|
# caller can tell "don't know" apart from "no". Also naturally reports
|
|
# "yes" if the selection changed since the last build (a country now
|
|
# relevant that wasn't recorded, or vice versa) — same reasoning
|
|
# terrain_update_available? below already uses.
|
|
def self.offline_update_available?(data_dir, fetch_last_modified: method(:fetch_geofabrik_last_modified))
|
|
meta = offline_meta(data_dir)
|
|
return nil unless meta
|
|
|
|
codes = selected_regions(data_dir)
|
|
return nil if codes.empty?
|
|
|
|
current = codes.map do |code|
|
|
slug = GEOFABRIK_SLUGS[code]
|
|
return nil unless slug
|
|
|
|
{ "code" => code, "last_modified" => fetch_last_modified.call(geofabrik_url(slug)) }
|
|
end.sort_by { |s| s["code"] }
|
|
recorded = (meta["sources"] || []).map { |s| s.slice("code", "last_modified") }.sort_by { |s| s["code"] }
|
|
current != recorded
|
|
rescue BuildError
|
|
nil
|
|
end
|
|
|
|
def self.fetch_geofabrik_last_modified(url, limit: 5)
|
|
raise BuildError, "too many redirects reaching #{url}" if limit.zero?
|
|
|
|
uri = URI(url)
|
|
response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.head(uri) }
|
|
case response
|
|
when Net::HTTPSuccess then response["last-modified"]
|
|
when Net::HTTPRedirection then fetch_geofabrik_last_modified(URI.join(url, response["location"]).to_s, limit: limit - 1)
|
|
else raise BuildError, "could not reach #{url}"
|
|
end
|
|
end
|
|
|
|
def self.offline_meta_path(data_dir)
|
|
File.join(data_dir, "offline.pmtiles.meta.json")
|
|
end
|
|
private_class_method :offline_meta_path
|
|
|
|
# Deletes offline.pmtiles, its sidecar meta, and the whole per-country
|
|
# download cache (data/geofabrik_cache/) — "start over" for the offline
|
|
# basemap. The current selection (data/offline_regions.json) is left
|
|
# alone, so a following build just re-downloads it fresh.
|
|
def self.delete_offline_data(data_dir)
|
|
OFFLINE_MUTEX.synchronize { raise BuildError, "a build is already running" if OFFLINE_STATUS[:state] == "running" }
|
|
|
|
File.delete(File.join(data_dir, "offline.pmtiles")) if File.exist?(File.join(data_dir, "offline.pmtiles"))
|
|
File.delete(offline_meta_path(data_dir)) if File.exist?(offline_meta_path(data_dir))
|
|
cache_dir = geofabrik_cache_dir(data_dir)
|
|
FileUtils.remove_entry(cache_dir) if Dir.exist?(cache_dir)
|
|
end
|
|
|
|
# Deletes just one country's cached .osm.pbf (and its Last-Modified
|
|
# stamp) — e.g. to force a fresh download without touching the others,
|
|
# or to reclaim space for a country no longer selected.
|
|
def self.delete_cached_extract(data_dir, code)
|
|
OFFLINE_MUTEX.synchronize { raise BuildError, "a build is already running" if OFFLINE_STATUS[:state] == "running" }
|
|
|
|
cache_dir = geofabrik_cache_dir(data_dir)
|
|
path = File.join(cache_dir, "#{code}.osm.pbf")
|
|
File.delete(path) if File.exist?(path)
|
|
File.delete("#{path}.last_modified") if File.exist?("#{path}.last_modified")
|
|
end
|
|
|
|
# What's actually sitting in the per-country download cache right now —
|
|
# not the same thing as offline_meta's "sources", which only reflects
|
|
# whatever the *last successful build* used. A country can be cached
|
|
# without ever having been built (selected, then deselected again
|
|
# before Apply) or built from without still being selected, and this is
|
|
# what lets the region picker show/clear either case.
|
|
def self.cached_extracts(data_dir)
|
|
cache_dir = geofabrik_cache_dir(data_dir)
|
|
return [] unless Dir.exist?(cache_dir)
|
|
|
|
Dir.glob(File.join(cache_dir, "*.osm.pbf")).map do |path|
|
|
stamp_path = "#{path}.last_modified"
|
|
{
|
|
"code" => File.basename(path, ".osm.pbf"),
|
|
"size" => File.size(path),
|
|
"last_modified" => File.exist?(stamp_path) ? File.read(stamp_path) : nil,
|
|
}
|
|
end.sort_by { |s| s["code"] }
|
|
end
|
|
|
|
# --- terrain.pmtiles ---
|
|
|
|
# planet.pmtiles (z0-12, always) plus whichever regional chunks
|
|
# intersect the selection's aggregate bbox (z13+, zero or more) are
|
|
# each `extract`ed on their own using the *same* --region= file, then
|
|
# combined with `pmtiles merge` if there was more than one. Always
|
|
# safe to merge regardless of how many countries are selected or
|
|
# whether they're adjacent: each Mapterhorn chunk is its own
|
|
# pre-partitioned, non-overlapping source grid cell, so two different
|
|
# chunks' extracts can never share a tile id — the only thing `merge`
|
|
# refuses (see pmtiles/merge.go's own disjointness check). What would
|
|
# NOT be safe is extracting the *same* source once per country and
|
|
# merging those — adjacent countries' region-extracts would share
|
|
# border tiles (extraction copies whole tiles, it doesn't clip vector
|
|
# content to the polygon) — so this always does one extract per
|
|
# *source archive*, never one per country.
|
|
def self.build_terrain_pmtiles(codes, data_dir, fetch_manifest, report_step)
|
|
report_step.call("fetching mapterhorn manifest")
|
|
items = fetch_manifest.call
|
|
planet = items.find { |item| item["name"] == MAPTERHORN_PLANET_NAME }
|
|
raise BuildError, "mapterhorn manifest is missing #{MAPTERHORN_PLANET_NAME}" unless planet
|
|
|
|
scratch_dir = Dir.mktmpdir("maptool-terrain")
|
|
south, west, north, east = region_bbox(codes)
|
|
region_path = File.join(scratch_dir, "region.geojson")
|
|
selected = region_features.select { |f| codes.include?(f["properties"]["iso_a2"]) }
|
|
File.write(region_path, JSON.generate(type: "FeatureCollection", features: selected))
|
|
|
|
chunks = items.select { |item| item["name"] != MAPTERHORN_PLANET_NAME && bbox_intersects?(item, south, west, north, east) }
|
|
|
|
# .tmp.pmtiles, not .pmtiles.tmp — same reasoning as offline.pmtiles'
|
|
# own tmp_path above, in case go-pmtiles also derives format from
|
|
# the extension for its "merge" output.
|
|
tmp_path = File.join(data_dir, "terrain.tmp.pmtiles")
|
|
sources = [planet, *chunks]
|
|
labels = sources.map { |item| item["name"] }
|
|
extracts = sources.each_with_index.map do |item, i|
|
|
report_step.call(progress_step("extracting #{item["name"]}", labels, i))
|
|
extract_path = File.join(scratch_dir, "extract-#{i}.pmtiles")
|
|
run_pmtiles!("extract", item["url"], extract_path, "--region=#{region_path}", "--maxzoom=#{TERRAIN_MAX_ZOOM}")
|
|
extract_path
|
|
end
|
|
|
|
if extracts.size == 1
|
|
File.rename(extracts.first, tmp_path)
|
|
else
|
|
report_step.call("merging with pmtiles")
|
|
run_pmtiles!("merge", *extracts, tmp_path)
|
|
end
|
|
File.rename(tmp_path, File.join(data_dir, "terrain.pmtiles"))
|
|
File.write(terrain_meta_path(data_dir), JSON.generate(sources: [planet, *chunks].map { |i| { name: i["name"], md5sum: i["md5sum"] } }))
|
|
ensure
|
|
FileUtils.remove_entry(scratch_dir) if scratch_dir && Dir.exist?(scratch_dir)
|
|
File.delete(tmp_path) if tmp_path && File.exist?(tmp_path)
|
|
end
|
|
private_class_method :build_terrain_pmtiles
|
|
|
|
def self.bbox_intersects?(item, south, west, north, east)
|
|
item["min_lon"] < east && item["max_lon"] > west && item["min_lat"] < north && item["max_lat"] > south
|
|
end
|
|
private_class_method :bbox_intersects?
|
|
|
|
def self.fetch_mapterhorn_manifest
|
|
response = Net::HTTP.get_response(URI(MAPTERHORN_MANIFEST_URL))
|
|
raise BuildError, "could not reach #{MAPTERHORN_MANIFEST_URL}" unless response.is_a?(Net::HTTPSuccess)
|
|
|
|
JSON.parse(response.body).fetch("items")
|
|
rescue JSON::ParserError
|
|
raise BuildError, "mapterhorn manifest was not valid JSON"
|
|
end
|
|
private_class_method :fetch_mapterhorn_manifest
|
|
|
|
# Sidecar recording which Mapterhorn sources (name + md5sum) the on-disk
|
|
# terrain.pmtiles was actually extracted+merged from. nil if never built.
|
|
def self.terrain_meta(data_dir)
|
|
read_json(terrain_meta_path(data_dir))
|
|
end
|
|
|
|
# Same "don't know" vs. "no" distinction as offline_update_available?
|
|
# above. Recomputes which sources *would* be selected for the current
|
|
# selection (same manifest + bbox_intersects? this build itself uses)
|
|
# and compares their md5sums against what's recorded — so this also
|
|
# correctly says "yes" if the selection changed since the last build
|
|
# (a chunk now relevant that wasn't recorded), not just if Mapterhorn's
|
|
# own data moved on.
|
|
def self.terrain_update_available?(data_dir, fetch_manifest: method(:fetch_mapterhorn_manifest))
|
|
meta = terrain_meta(data_dir)
|
|
return nil unless meta
|
|
|
|
codes = selected_regions(data_dir)
|
|
return nil if codes.empty?
|
|
|
|
items = fetch_manifest.call
|
|
planet = items.find { |item| item["name"] == MAPTERHORN_PLANET_NAME }
|
|
return nil unless planet
|
|
|
|
south, west, north, east = region_bbox(codes)
|
|
chunks = items.select { |item| item["name"] != MAPTERHORN_PLANET_NAME && bbox_intersects?(item, south, west, north, east) }
|
|
current = [planet, *chunks].map { |i| { "name" => i["name"], "md5sum" => i["md5sum"] } }.sort_by { |s| s["name"] }
|
|
current != (meta["sources"] || []).sort_by { |s| s["name"] }
|
|
rescue BuildError
|
|
nil
|
|
end
|
|
|
|
def self.terrain_meta_path(data_dir)
|
|
File.join(data_dir, "terrain.pmtiles.meta.json")
|
|
end
|
|
private_class_method :terrain_meta_path
|
|
|
|
# Deletes terrain.pmtiles and its sidecar meta — same "start over"
|
|
# reasoning as delete_offline_data above. No download cache to clean up
|
|
# here: terrain.pmtiles' pmtiles-extract step reads its sources over
|
|
# HTTP range requests rather than downloading a whole archive first.
|
|
def self.delete_terrain_data(data_dir)
|
|
TERRAIN_MUTEX.synchronize { raise BuildError, "a build is already running" if TERRAIN_STATUS[:state] == "running" }
|
|
|
|
File.delete(File.join(data_dir, "terrain.pmtiles")) if File.exist?(File.join(data_dir, "terrain.pmtiles"))
|
|
File.delete(terrain_meta_path(data_dir)) if File.exist?(terrain_meta_path(data_dir))
|
|
end
|
|
|
|
# --- shared ---
|
|
|
|
def self.read_json(path)
|
|
File.exist?(path) ? JSON.parse(File.read(path)) : nil
|
|
rescue JSON::ParserError
|
|
nil
|
|
end
|
|
private_class_method :read_json
|
|
|
|
# ENV read lazily (not load-time constants) so tests can point these at
|
|
# fake binaries without a require-order dance.
|
|
def self.pmtiles_bin
|
|
ENV.fetch("PMTILES_BIN", "pmtiles")
|
|
end
|
|
|
|
def self.osmium_bin
|
|
ENV.fetch("OSMIUM_BIN", "osmium")
|
|
end
|
|
|
|
def self.java_bin
|
|
ENV.fetch("JAVA_BIN", "java")
|
|
end
|
|
|
|
def self.planetiler_jar
|
|
ENV.fetch("PLANETILER_JAR", "/usr/local/lib/planetiler.jar")
|
|
end
|
|
|
|
# Some CLI tools (go-pmtiles included) write their actual error to
|
|
# stdout rather than stderr — combine both rather than risk raising a
|
|
# BuildError with nothing useful in it.
|
|
def self.command_output(stdout, stderr)
|
|
[stdout, stderr].map(&:strip).reject(&:empty?).join("\n")
|
|
end
|
|
private_class_method :command_output
|
|
|
|
# extract/merge stream tens of GB over HTTP range requests; retry a
|
|
# transient failure (e.g. a mid-transfer HTTP/2 stream reset from the
|
|
# remote) a couple times before giving up, rather than forcing a full
|
|
# re-download on every hiccup.
|
|
def self.run_pmtiles!(*args, attempts: 3)
|
|
attempts.times do |i|
|
|
stdout, stderr, status = Open3.capture3(pmtiles_bin, *args)
|
|
return if status.success?
|
|
raise BuildError, "pmtiles exited with status #{status.exitstatus}: #{command_output(stdout, stderr)}" if i == attempts - 1
|
|
end
|
|
rescue Errno::ENOENT, Errno::EACCES => e
|
|
raise BuildError, "could not run '#{pmtiles_bin}' (#{e.message}) — is go-pmtiles installed and on PATH? See docs/development.adoc."
|
|
end
|
|
private_class_method :run_pmtiles!
|
|
|
|
def self.run_osmium!(*args)
|
|
stdout, stderr, status = Open3.capture3(osmium_bin, *args)
|
|
raise BuildError, "osmium exited with status #{status.exitstatus}: #{command_output(stdout, stderr)}" unless status.success?
|
|
rescue Errno::ENOENT, Errno::EACCES => e
|
|
raise BuildError, "could not run '#{osmium_bin}' (#{e.message}) — is osmium-tool installed and on PATH? See docs/development.adoc."
|
|
end
|
|
private_class_method :run_osmium!
|
|
|
|
def self.run_planetiler!(*args)
|
|
stdout, stderr, status = Open3.capture3(java_bin, "-jar", planetiler_jar, *args)
|
|
raise BuildError, "planetiler exited with status #{status.exitstatus}: #{command_output(stdout, stderr)}" unless status.success?
|
|
rescue Errno::ENOENT, Errno::EACCES => e
|
|
raise BuildError, "could not run '#{java_bin} -jar #{planetiler_jar}' (#{e.message}) — is Java and planetiler.jar installed? See docs/development.adoc."
|
|
end
|
|
private_class_method :run_planetiler!
|
|
end
|
|
end
|