From f89dea58055408f9674db907a5ac30240c648fe0 Mon Sep 17 00:00:00 2001 From: Roman Priesol Date: Fri, 11 Sep 2026 12:26:03 +0200 Subject: [PATCH] Build terrain.pmtiles from Mapterhorn instead of fetching tiles ourselves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mapterhorn (mapterhorn.com, Protomaps-affiliated) publishes pre-built Terrarium-encoded terrain as PMTiles: a global planet.pmtiles (z0-12) plus ~450 regional chunks (z13+). Same pmtiles extract --bbox=... mechanism offline.pmtiles already uses, just against their files — confirmed live against their real manifest and the documented Interlaken example. Replaces fetching every individual tile ourselves (even after this session's earlier concurrency fix, still tens of minutes for a real region) with 1-3 extract calls plus a merge when the bbox spans more than one chunk: seconds, not minutes. Removes CONCURRENT_TILE_FETCHES, write_terrarium_mbtiles, tile_range, lng/lat_to_tile_x/y, fetch_terrarium_tile, and the sqlite3 gem entirely — none of it is needed once the tiles arrive pre-packaged. dem.js's point-elevation lookup (shadow.js's terrain-shadow overlay) hardcoded a 256px PNG tile assumption that Mapterhorn's 512px WebP tiles would have silently broken — now reads the actually-decoded tile's own size and mime type instead of assuming either. --- Gemfile | 1 - Gemfile.lock | 21 ---- docs/development.adoc | 40 +++++--- docs/third-party-notices.adoc | 9 ++ lib/maptool/pmtiles_builder.rb | 182 ++++++++++----------------------- public/js/dem.js | 36 ++++--- public/js/overlays.js | 12 ++- test/test_pmtiles_builder.rb | 157 ++++++++++++++++++---------- 8 files changed, 224 insertions(+), 234 deletions(-) diff --git a/Gemfile b/Gemfile index 99f2194..c529183 100644 --- a/Gemfile +++ b/Gemfile @@ -6,7 +6,6 @@ gem "puma", "~> 8.0" gem "rexml", "~> 3.4" gem "oauth2", "~> 2.0" gem "rubyzip", "~> 2.4" -gem "sqlite3", "~> 2.0" group :development do gem "rerun" diff --git a/Gemfile.lock b/Gemfile.lock index c4dfae9..db423d5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -95,16 +95,6 @@ GEM snaky_hash (2.0.7) hashie (>= 0.1.0, < 6) version_gem (~> 1.1, >= 1.1.14) - sqlite3 (2.9.6-aarch64-linux-gnu) - sqlite3 (2.9.6-aarch64-linux-musl) - sqlite3 (2.9.6-arm-linux-gnu) - sqlite3 (2.9.6-arm-linux-musl) - sqlite3 (2.9.6-arm64-darwin) - sqlite3 (2.9.6-x86-linux-gnu) - sqlite3 (2.9.6-x86-linux-musl) - sqlite3 (2.9.6-x86_64-darwin) - sqlite3 (2.9.6-x86_64-linux-gnu) - sqlite3 (2.9.6-x86_64-linux-musl) tilt (2.9.0) uri (1.1.1) version_gem (1.1.15) @@ -132,7 +122,6 @@ DEPENDENCIES rubyzip (~> 2.4) sinatra (~> 4.2) sinatra-contrib (~> 4.2) - sqlite3 (~> 2.0) CHECKSUMS anonymous_loader (0.1.3) sha256=084a18e2439144d955447dc11dfc982f41fcd1583ad32d4d55151325dc44cb55 @@ -182,16 +171,6 @@ CHECKSUMS sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 sinatra-contrib (4.2.1) sha256=10d091c944d268aa910c618ea40a3c3ebe0533e6e32990d84af92235a3d26b4a snaky_hash (2.0.7) sha256=7d02c70012a3f932e48860cd024577908300c9aa615e0cb9b450aaa749cbcb4d - sqlite3 (2.9.6-aarch64-linux-gnu) sha256=d8b1f7d23efd7abac285775a9566562fc7debfef79d594e3a20354406fb7907c - sqlite3 (2.9.6-aarch64-linux-musl) sha256=3579e1c98cdc7ff5c3722847bb63ed4e1efb7ff675cb5e1e48ef2d4da5fb3bc9 - sqlite3 (2.9.6-arm-linux-gnu) sha256=33541500e3615da02afe54a9cc38b17a6985d3cf9d8b76d6d0a83002f114e7ec - sqlite3 (2.9.6-arm-linux-musl) sha256=c5490af48bb228fefa54314e9541375c3907e70f8109f3881b5ff97e1c93ae33 - sqlite3 (2.9.6-arm64-darwin) sha256=849b5d7f795e60fe25076d62c72dd722beb45b3850b516ad978d60ee848ec15b - sqlite3 (2.9.6-x86-linux-gnu) sha256=fbaa9f46f9708f57dd8a459b37fc269f9613e0cacf1547df01aa439cc45c20c0 - sqlite3 (2.9.6-x86-linux-musl) sha256=6715026fbb5530e810b28ef43b9c4f84cd3c991f67b9d808a31fcc32b847abbd - sqlite3 (2.9.6-x86_64-darwin) sha256=b5842fea77781c14da03fa7bc0feb82db03a69e135affcb6f5399cbd2797a5f3 - sqlite3 (2.9.6-x86_64-linux-gnu) sha256=613188ce02f614126ddbc38c5e217ccffd6306d0dcd9adca9764547aa890a634 - sqlite3 (2.9.6-x86_64-linux-musl) sha256=d493b11818a3573387a1d56e1ee8fa00da23a683a7a1cc063e7a0feeed843abf tilt (2.9.0) sha256=da5735d0280bba96e9a91041bb14aee435ccad5c17b0fa519249ae543d9aa3a5 uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 version_gem (1.1.15) sha256=a73241587b29252e3567e0c818ded80730eb754d43b508f6ce4db22ca9ba27d0 diff --git a/docs/development.adoc b/docs/development.adoc index ed849dd..bb0a85a 100644 --- a/docs/development.adoc +++ b/docs/development.adoc @@ -58,13 +58,28 @@ no API key). For it to work offline too, place a PMTiles archive of Terrarium-encoded `raster-dem` tiles at `data/terrain.pmtiles` (`GET /api/terrain-tiles` reports `available: true` once it's there, same pattern as offline map data above). Also generatable in-app (see -below) or by hand — build one locally from a DEM source (e.g. -Copernicus GLO-30 or SRTM) with a tool like -https://github.com/mapbox/rio-rgbify[rio-rgbify] (GeoTIFF -> -Terrarium-encoded PNG tiles in an `.mbtiles`), then -`pmtiles convert out.mbtiles data/terrain.pmtiles` with -https://github.com/protomaps/go-pmtiles[go-pmtiles], same as -`data/offline.pmtiles`. +below) or by hand, the same `go-pmtiles extract` mechanism as +`data/offline.pmtiles` above — but from +https://mapterhorn.com[Mapterhorn], a Protomaps-affiliated project +publishing pre-built Terrarium-encoded terrain as PMTiles (a global +`planet.pmtiles`, z0-12, plus ~450 regional chunks covering z13 and up +— see https://mapterhorn.com/data-access/[their own data-access page] +for the full manifest and per-source license/attribution details, CC-BY-4.0 +and similar, varying by region): + +[source,shell] +---- +pmtiles extract https://download.mapterhorn.com/planet.pmtiles planet-extract.pmtiles --bbox=west,south,east,north --maxzoom=15 +# only if a regional chunk covers the bbox too (see download_urls.json) — +# most bboxes hit exactly one, a bbox straddling a chunk boundary hits two: +pmtiles extract https://download.mapterhorn.com/.pmtiles chunk-extract.pmtiles --bbox=west,south,east,north --maxzoom=15 +pmtiles merge planet-extract.pmtiles chunk-extract.pmtiles data/terrain.pmtiles +---- + +Tiles are 512px WebP, not the 256px PNG this app's *online* Terrarium +source uses — `public/js/dem.js`'s own point-elevation lookups +(`shadow.js`'s terrain-shadow overlay) read the actual decoded tile +size rather than assuming one, so this isn't a problem either way. == In-app regeneration @@ -78,11 +93,12 @@ wraps exactly the manual `go-pmtiles` workflows documented above (`lib/maptool/pmtiles_builder.rb`) — the server needs the https://github.com/protomaps/go-pmtiles[go-pmtiles] CLI installed and on `PATH` (override with the `PMTILES_BIN` env var if it lives -somewhere else); regenerating terrain.pmtiles this way fetches -individual Terrarium tiles itself and packs them into an MBTiles file -(the `sqlite3` gem) before the same `pmtiles convert` step, since no -public Terrarium-encoded PMTiles archive exists to `extract` from the -way offline.pmtiles above can. +somewhere else). Fetches Mapterhorn's own small JSON manifest to work +out which regional chunk(s), if any, intersect the bbox, `extract`s +each (plus `planet.pmtiles`, always), then `merge`s them if there was +more than one — typically seconds, not the tens of minutes an earlier +version of this (fetching every individual Terrarium tile itself) used +to take for a real-sized region. A build runs in the background (`POST /api/offline-tiles/build` / `/api/terrain-tiles/build`, polled via the matching `*/build-status` diff --git a/docs/third-party-notices.adoc b/docs/third-party-notices.adoc index 8a4f487..baa35af 100644 --- a/docs/third-party-notices.adoc +++ b/docs/third-party-notices.adoc @@ -82,6 +82,15 @@ The online map style (not vendored, fetched at runtime when the https://tiles.openfreemap.org/styles/liberty[OpenFreeMap "liberty"], built from https://www.openstreetmap.org[OpenStreetMap] data. +`terrain.pmtiles` (`lib/maptool/pmtiles_builder.rb`'s in-app +regeneration, or the manual recipe in `docs/development.adoc`) is +extracted at build time, not vendored, from +https://mapterhorn.com[Mapterhorn] — a Protomaps-affiliated project +publishing pre-built Terrarium-encoded terrain as PMTiles. Per-region +source licenses vary (CC-BY-4.0 and similar); see +https://download.mapterhorn.com/attribution.json[their own attribution +list] for the full provenance. + Not a frontend asset like the above, but also baked into this app's own Docker image (not a separate container the way kiwix-serve/gpsd are — see `docs/deployment.adoc`/`docs/development.adoc`): the `pmtiles` CLI diff --git a/lib/maptool/pmtiles_builder.rb b/lib/maptool/pmtiles_builder.rb index b913606..eab9553 100644 --- a/lib/maptool/pmtiles_builder.rb +++ b/lib/maptool/pmtiles_builder.rb @@ -2,8 +2,8 @@ require "open3" require "net/http" require "uri" require "json" -require "sqlite3" require "tmpdir" +require "fileutils" module Maptool # Regenerates data/offline.pmtiles and data/terrain.pmtiles for a given @@ -22,19 +22,20 @@ module Maptool PROTOMAPS_BUILDS_INDEX = "https://build-metadata.protomaps.dev/builds.json" PROTOMAPS_BUILD_BASE = "https://build.protomaps.com" - # Same public, no-API-key Terrarium tile source public/js/overlays.js's - # ONLINE_TERRAIN_TILES already fetches client-side — no shared constant - # between Ruby and JS, just the same literal URL template. Matches that - # source's own online maxzoom. - TERRARIUM_URL_TEMPLATE = "https://s3.amazonaws.com/elevation-tiles-prod/terrarium/%d/%d/%d.png" + # Mapterhorn (a Protomaps-affiliated project) publishes Terrarium- + # encoded terrain as pre-built PMTiles archives: one global + # planet.pmtiles (z0-12) plus ~450 regional chunks (z13 up to a + # per-region max, 13-18) — same "extract only the bbox's byte ranges + # from a remote archive" story as offline.pmtiles above, so no more + # fetching individual tiles or building an MBTiles file ourselves the + # way an earlier version of this file did. + MAPTERHORN_MANIFEST_URL = "https://download.mapterhorn.com/download_urls.json" + MAPTERHORN_PLANET_NAME = "planet.pmtiles" + # Matches the online hillshade layer's own cap (public/js/overlays.js's + # ONLINE_TERRAIN_TILES source, maxzoom: 15) — applied uniformly even + # though some Mapterhorn chunks natively offer more, so the offline + # experience doesn't vary in detail from one region to the next. TERRAIN_MAX_ZOOM = 15 - # A real bbox at z0..15 is tens of thousands of tiles — serial fetches - # (the original ponytail'd version of this) measured ~0.65s/tile, almost - # all TCP+TLS handshake, no connection reuse: hours for a modest region. - # This many workers, each keeping its own persistent connection (see - # fetch_terrarium_tile), is a big enough win to matter without looking - # like abuse against a public bucket. - CONCURRENT_TILE_FETCHES = 16 OFFLINE_STATUS = { state: "idle" } TERRAIN_STATUS = { state: "idle" } @@ -71,16 +72,12 @@ module Maptool Thread.new { run_job(OFFLINE_MUTEX, OFFLINE_STATUS) { extract_offline(bounds, data_dir, fetch_build_url) } } end - # fetch_tile: same injectable-seam reasoning as fetch_build_url above, - # for the many individual Terrarium tile fetches this needs instead of - # one index lookup. This build can run for minutes with nothing else to - # show for it, so write_terrarium_mbtiles reports tiles_done/tiles_total - # into the status Hash as it goes — otherwise a poller sees "running" - # the whole time with no way to tell it apart from stuck. - def self.build_terrain(bounds, data_dir:, fetch_tile: method(:fetch_terrarium_tile)) + # fetch_manifest: same injectable-seam reasoning as fetch_build_url + # above, for the one Mapterhorn manifest lookup this needs instead of + # the Protomaps build index. + def self.build_terrain(bounds, data_dir:, fetch_manifest: method(:fetch_mapterhorn_manifest)) begin_run!(TERRAIN_MUTEX, TERRAIN_STATUS) - progress = ->(done, total) { TERRAIN_MUTEX.synchronize { TERRAIN_STATUS.merge!(tiles_done: done, tiles_total: total) } } - Thread.new { run_job(TERRAIN_MUTEX, TERRAIN_STATUS) { build_terrain_pmtiles(bounds, data_dir, fetch_tile, progress) } } + Thread.new { run_job(TERRAIN_MUTEX, TERRAIN_STATUS) { build_terrain_pmtiles(bounds, data_dir, fetch_manifest) } } end def self.begin_run!(mutex, status) @@ -128,122 +125,55 @@ module Maptool # --- terrain.pmtiles --- - def self.build_terrain_pmtiles(bounds, data_dir, fetch_tile, progress) + # planet.pmtiles (z0-12, always) plus whichever regional chunks + # intersect the bbox (z13+, zero or more — most bboxes hit exactly one, + # a bbox straddling a chunk boundary hits two) are each `extract`ed on + # their own, then combined with `pmtiles merge` if there was more than + # one (planet + a chunk are disjoint by zoom level, so this is always a + # valid merge; go-pmtiles requires clustered, matching-tile-type inputs, + # which every Mapterhorn PMTiles archive already satisfies). + def self.build_terrain_pmtiles(bounds, data_dir, fetch_manifest) south, west, north, east = bounds - tmp_mbtiles = File.join(Dir.tmpdir, "maptool-terrain-#{Process.pid}-#{Time.now.to_i}.mbtiles") - tmp_path = File.join(data_dir, "terrain.pmtiles.tmp") + items = fetch_manifest.call + planet = items.find { |item| item["name"] == MAPTERHORN_PLANET_NAME } + raise BuildError, "mapterhorn manifest is missing #{MAPTERHORN_PLANET_NAME}" unless planet - write_terrarium_mbtiles(tmp_mbtiles, south, west, north, east, fetch_tile, progress) - run_pmtiles!("convert", tmp_mbtiles, tmp_path) + chunks = items.select { |item| item["name"] != MAPTERHORN_PLANET_NAME && bbox_intersects?(item, south, west, north, east) } + + scratch_dir = Dir.mktmpdir("maptool-terrain") + tmp_path = File.join(data_dir, "terrain.pmtiles.tmp") + extracts = [planet, *chunks].each_with_index.map do |item, i| + extract_path = File.join(scratch_dir, "extract-#{i}.pmtiles") + run_pmtiles!("extract", item["url"], extract_path, "--bbox=#{west},#{south},#{east},#{north}", "--maxzoom=#{TERRAIN_MAX_ZOOM}") + extract_path + end + + if extracts.size == 1 + File.rename(extracts.first, tmp_path) + else + run_pmtiles!("merge", *extracts, tmp_path) + end File.rename(tmp_path, File.join(data_dir, "terrain.pmtiles")) ensure - File.delete(tmp_mbtiles) if tmp_mbtiles && File.exist?(tmp_mbtiles) + FileUtils.remove_entry(scratch_dir) if scratch_dir && Dir.exist?(scratch_dir) File.delete(tmp_path) if tmp_path && File.exist?(tmp_path) end private_class_method :build_terrain_pmtiles - # Terrarium PNG tiles for z0..TERRAIN_MAX_ZOOM over the bbox, packed - # into a minimal MBTiles (SQLite) archive — go-pmtiles' own `convert` - # only accepts MBTiles/PMTiles as input, no raw tile directory, and no - # public Terrarium-encoded PMTiles archive is known to exist to - # `extract` from the way offline.pmtiles does above. - # - # fetch_tile.call happens on CONCURRENT_TILE_FETCHES worker threads at - # once (I/O-bound — Ruby's GVL is released while a thread blocks on - # network I/O, so this genuinely overlaps); every SQLite write and - # progress update stays on this thread, draining a results Queue the - # workers feed, so nothing here needs its own locking. - def self.write_terrarium_mbtiles(path, south, west, north, east, fetch_tile, progress) - db = SQLite3::Database.new(path) - db.execute_batch(<<~SQL) - CREATE TABLE metadata (name TEXT, value TEXT); - CREATE TABLE tiles (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_data BLOB); - CREATE UNIQUE INDEX tile_index ON tiles (zoom_level, tile_column, tile_row); - SQL - { "format" => "png", "name" => "terrain", "type" => "baselayer", "minzoom" => "0", "maxzoom" => TERRAIN_MAX_ZOOM.to_s }.each do |name, value| - db.execute("INSERT INTO metadata (name, value) VALUES (?, ?)", [name, value]) - end - - # tile_range per zoom computed once upfront (cheap — no fetching, just - # tile math), so progress can report a real total from the very first - # call instead of only once the run finishes. - work = (0..TERRAIN_MAX_ZOOM).flat_map { |z| tile_range(z, south, west, north, east).map { |x, y| [z, x, y] } } - total = work.size - done = 0 - progress.call(done, total) - - queue = Queue.new - work.each { |item| queue << item } - CONCURRENT_TILE_FETCHES.times { queue << nil } # one stop signal per worker - - results = Queue.new - workers = CONCURRENT_TILE_FETCHES.times.map do - Thread.new do - while (item = queue.pop) - z, x, y = item - results << [z, x, y, fetch_tile.call(z, x, y)] - end - ensure - # Only set when fetch_tile is fetch_terrarium_tile (a test fake - # has no reason to touch this) — closes this worker's persistent - # connection rather than leaving the socket for GC to eventually - # notice. - Thread.current[:maptool_terrarium_http]&.finish - end - end - - total.times do - z, x, y, bytes = results.pop - done += 1 - progress.call(done, total) - next unless bytes - - # MBTiles uses TMS row numbering (0 at the south) — the opposite of - # the XYZ scheme both the Terrarium source and this app's own online - # raster-dem layer use (0 at the north) — flipped here so go-pmtiles - # ends up serving the same tile at the same z/x/y this app already - # requests online. - tms_y = (2**z) - 1 - y - db.execute("INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", [z, x, tms_y, bytes.b]) - end - workers.each(&:join) - ensure - db&.close + def self.bbox_intersects?(item, south, west, north, east) + item["min_lon"] < east && item["max_lon"] > west && item["min_lat"] < north && item["max_lat"] > south end - private_class_method :write_terrarium_mbtiles + private_class_method :bbox_intersects? - def self.tile_range(z, south, west, north, east) - n = 2**z - x_min, x_max = [lng_to_tile_x(west, n), lng_to_tile_x(east, n)].minmax - y_min, y_max = [lat_to_tile_y(north, n), lat_to_tile_y(south, n)].minmax - x_min.upto(x_max).flat_map { |x| y_min.upto(y_max).map { |y| [x, y] } } - end - private_class_method :tile_range + def self.fetch_mapterhorn_manifest + response = Net::HTTP.get_response(URI(MAPTERHORN_MANIFEST_URL)) + raise BuildError, "could not reach #{MAPTERHORN_MANIFEST_URL}" unless response.is_a?(Net::HTTPSuccess) - def self.lng_to_tile_x(lng, n) - (((lng + 180) / 360) * n).floor.clamp(0, n - 1) + JSON.parse(response.body).fetch("items") + rescue JSON::ParserError + raise BuildError, "mapterhorn manifest was not valid JSON" end - private_class_method :lng_to_tile_x - - def self.lat_to_tile_y(lat, n) - lat_rad = lat * Math::PI / 180 - (((1 - (Math.log(Math.tan(lat_rad) + (1 / Math.cos(lat_rad))) / Math::PI)) / 2) * n).floor.clamp(0, n - 1) - end - private_class_method :lat_to_tile_y - - # One persistent connection per calling Thread (each of - # write_terrarium_mbtiles' worker threads calls this many times) instead - # of Net::HTTP.get_response's usual "new TCP+TLS handshake per call" — - # that handshake was most of the ~0.65s/tile this used to cost. - def self.fetch_terrarium_tile(z, x, y) - uri = URI(format(TERRARIUM_URL_TEMPLATE, z: z, x: x, y: y)) - http = (Thread.current[:maptool_terrarium_http] ||= Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https")) - response = http.get(uri.request_uri) - response.is_a?(Net::HTTPSuccess) ? response.body : nil - rescue StandardError - nil - end - private_class_method :fetch_terrarium_tile + private_class_method :fetch_mapterhorn_manifest # --- shared --- diff --git a/public/js/dem.js b/public/js/dem.js index eee9ffa..bd158f1 100644 --- a/public/js/dem.js +++ b/public/js/dem.js @@ -7,11 +7,14 @@ import { ONLINE_TERRAIN_TILES } from "/js/overlays.js"; const DEM_ZOOM = 10; // fixed lookup zoom, independent of the map's own display zoom -const TILE_SIZE = 256; -const tileCache = new Map(); // "z/x/y" -> Promise, so concurrent lookups on the same not-yet-fetched tile share one fetch +const tileCache = new Map(); // "z/x/y" -> Promise<{elevations,size}|null>, so concurrent lookups on the same not-yet-fetched tile share one fetch let offlinePmtiles = null; +// fracX/fracY (not pixel coordinates): the tile hasn't been fetched yet at +// this point, so its actual pixel size isn't known — online tiles are +// 256px PNG, offline ones (Mapterhorn, see lib/maptool/pmtiles_builder.rb) +// are 512px WebP. elevationAt scales these once the real size is known. function lngLatToTile(lng, lat, zoom) { const n = 2 ** zoom; const x = ((lng + 180) / 360) * n; @@ -19,16 +22,11 @@ function lngLatToTile(lng, lat, zoom) { const y = ((1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI) / 2) * n; const tileX = Math.floor(x); const tileY = Math.floor(y); - return { - tileX, - tileY, - px: Math.min(TILE_SIZE - 1, Math.floor((x - tileX) * TILE_SIZE)), - py: Math.min(TILE_SIZE - 1, Math.floor((y - tileY) * TILE_SIZE)), - }; + return { tileX, tileY, fracX: x - tileX, fracY: y - tileY }; } -async function decodeTerrarium(pngBytes) { - const bitmap = await createImageBitmap(new Blob([pngBytes], { type: "image/png" })); +async function decodeTerrarium(bytes, mime) { + const bitmap = await createImageBitmap(new Blob([bytes], { type: mime })); const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); const ctx = canvas.getContext("2d"); ctx.drawImage(bitmap, 0, 0); @@ -37,18 +35,18 @@ async function decodeTerrarium(pngBytes) { for (let i = 0; i < elevations.length; i++) { elevations[i] = data[i * 4] * 256 + data[i * 4 + 1] + data[i * 4 + 2] / 256 - 32768; } - return elevations; + return { elevations, size: bitmap.width }; } async function fetchTileBytes(z, x, y, offlineAvailable) { if (offlineAvailable) { if (!offlinePmtiles) offlinePmtiles = new window.pmtiles.PMTiles(`${window.location.origin}/tiles/terrain.pmtiles`); const result = await offlinePmtiles.getZxy(z, x, y); - return result ? new Uint8Array(result.data) : null; + return result ? { bytes: new Uint8Array(result.data), mime: "image/webp" } : null; } const url = ONLINE_TERRAIN_TILES[0].replace("{z}", z).replace("{x}", x).replace("{y}", y); const response = await fetch(url); - return response.ok ? new Uint8Array(await response.arrayBuffer()) : null; + return response.ok ? { bytes: new Uint8Array(await response.arrayBuffer()), mime: "image/png" } : null; } function fetchTile(z, x, y, offlineAvailable) { @@ -57,7 +55,7 @@ function fetchTile(z, x, y, offlineAvailable) { if (cached) return cached; const promise = fetchTileBytes(z, x, y, offlineAvailable) - .then((bytes) => (bytes ? decodeTerrarium(bytes) : null)) + .then((result) => (result ? decodeTerrarium(result.bytes, result.mime) : null)) .catch(() => null); tileCache.set(key, promise); return promise; @@ -66,7 +64,11 @@ function fetchTile(z, x, y, offlineAvailable) { // -> Promise — null on a fetch/decode failure, or a point // falling outside a partial offline extract's zoom range. export async function elevationAt(lng, lat, offlineAvailable) { - const { tileX, tileY, px, py } = lngLatToTile(lng, lat, DEM_ZOOM); - const elevations = await fetchTile(DEM_ZOOM, tileX, tileY, offlineAvailable); - return elevations ? elevations[py * TILE_SIZE + px] : null; + const { tileX, tileY, fracX, fracY } = lngLatToTile(lng, lat, DEM_ZOOM); + const tile = await fetchTile(DEM_ZOOM, tileX, tileY, offlineAvailable); + if (!tile) return null; + + const px = Math.min(tile.size - 1, Math.floor(fracX * tile.size)); + const py = Math.min(tile.size - 1, Math.floor(fracY * tile.size)); + return tile.elevations[py * tile.size + px]; } diff --git a/public/js/overlays.js b/public/js/overlays.js index b2c366c..43a9e5d 100644 --- a/public/js/overlays.js +++ b/public/js/overlays.js @@ -80,8 +80,18 @@ class HillshadeControl { _addLayer() { if (this._map.getSource(HILLSHADE_ID)) return; + // terrain.pmtiles is built from Mapterhorn (mapterhorn.com — see + // lib/maptool/pmtiles_builder.rb) whether generated in-app or by hand + // (docs/development.adoc), so this credits them regardless of how it + // got there — their own site lists the full per-region source + // provenance (CC-BY-4.0 and similar), not practical to reproduce here. const source = this._offlineAvailable - ? { type: "raster-dem", url: `pmtiles://${window.location.origin}/tiles/terrain.pmtiles`, encoding: "terrarium" } + ? { + type: "raster-dem", + url: `pmtiles://${window.location.origin}/tiles/terrain.pmtiles`, + encoding: "terrarium", + attribution: "Terrain: Mapterhorn", + } : { type: "raster-dem", tiles: ONLINE_TERRAIN_TILES, diff --git a/test/test_pmtiles_builder.rb b/test/test_pmtiles_builder.rb index e6b9798..afb5cc3 100644 --- a/test/test_pmtiles_builder.rb +++ b/test/test_pmtiles_builder.rb @@ -3,22 +3,37 @@ require "maptool/pmtiles_builder" require "tmpdir" require "fileutils" -# Network calls (the protomaps build index, individual Terrarium tile -# fetches) are never exercised here — same "keep this offline/deterministic" -# reasoning test_helper.rb already gives for /api/plugins staying off the -# real Overpass API — build_offline/build_terrain take fetch_build_url/ -# fetch_tile precisely so tests can swap those out. A fake `pmtiles` CLI -# (PMTILES_BIN) stands in for the real go-pmtiles binary. +# Network calls (the protomaps build index, the mapterhorn manifest) are +# never exercised here — same "keep this offline/deterministic" reasoning +# test_helper.rb already gives for /api/plugins staying off the real +# Overpass API — build_offline/build_terrain take fetch_build_url/ +# fetch_manifest precisely so tests can swap those out. A fake `pmtiles` +# CLI (PMTILES_BIN) stands in for the real go-pmtiles binary, logging each +# invocation's subcommand + first arg to PMTILES_LOG so tests can assert on +# what actually got called (e.g. "was merge invoked, and with what"). class PmtilesBuilderTest < Minitest::Test def setup @data_dir = Dir.mktmpdir("maptool-pmtiles-test-data") @bin_dir = Dir.mktmpdir("maptool-pmtiles-test-bin") + @log = File.join(@bin_dir, "pmtiles.log") - # $3 is the output path either way: `extract SOURCE OUTPUT --bbox=... - # --maxzoom=...` and `convert INPUT OUTPUT` both put it 3rd, even though - # it's not the *last* arg in the extract case (the --flags follow it). + # Output path: $3 for `extract SOURCE OUTPUT --bbox=... --maxzoom=...` + # and `convert INPUT OUTPUT` alike; the *last* arg for `merge INPUT_1 + # [INPUT_2 ...] OUTPUT`, since its input count varies. @pmtiles_ok = File.join(@bin_dir, "pmtiles-ok") - File.write(@pmtiles_ok, "#!/bin/sh\necho fake > \"$3\"\n") + File.write(@pmtiles_ok, <<~SH) + #!/bin/sh + echo "$1 $2" >> "$PMTILES_LOG" + case "$1" in + merge) + for last; do :; done + echo fake > "$last" + ;; + *) + echo fake > "$3" + ;; + esac + SH File.chmod(0o755, @pmtiles_ok) @pmtiles_fail = File.join(@bin_dir, "pmtiles-fail") @@ -26,6 +41,7 @@ class PmtilesBuilderTest < Minitest::Test File.chmod(0o755, @pmtiles_fail) @original_pmtiles_bin = ENV["PMTILES_BIN"] + ENV["PMTILES_LOG"] = @log Maptool::PmtilesBuilder.reset! end @@ -33,9 +49,26 @@ class PmtilesBuilderTest < Minitest::Test FileUtils.remove_entry(@data_dir) FileUtils.remove_entry(@bin_dir) ENV["PMTILES_BIN"] = @original_pmtiles_bin + ENV.delete("PMTILES_LOG") Maptool::PmtilesBuilder.reset! end + def log_lines + File.exist?(@log) ? File.readlines(@log, chomp: true) : [] + end + + # planet.pmtiles always covers the whole world (z0-12); chunk-a/chunk-b + # are disjoint small regions (z13+), matching Mapterhorn's real shape + # (see download_urls.json) closely enough to exercise bbox_intersects? + # and the extract-then-maybe-merge logic without a real manifest fetch. + def fake_manifest + [ + { "name" => "planet.pmtiles", "url" => "https://example.com/planet.pmtiles", "min_lon" => -180.0, "min_lat" => -85.0, "max_lon" => 180.0, "max_lat" => 85.0, "min_zoom" => 0, "max_zoom" => 12 }, + { "name" => "chunk-a.pmtiles", "url" => "https://example.com/chunk-a.pmtiles", "min_lon" => 17.0, "min_lat" => 48.0, "max_lon" => 17.5, "max_lat" => 48.5, "min_zoom" => 13, "max_zoom" => 15 }, + { "name" => "chunk-b.pmtiles", "url" => "https://example.com/chunk-b.pmtiles", "min_lon" => 20.0, "min_lat" => 50.0, "max_lon" => 20.5, "max_lat" => 50.5, "min_zoom" => 13, "max_zoom" => 15 }, + ] + end + def test_offline_and_terrain_status_start_idle assert_equal({ state: "idle" }, Maptool::PmtilesBuilder.offline_status) assert_equal({ state: "idle" }, Maptool::PmtilesBuilder.terrain_status) @@ -100,65 +133,77 @@ class PmtilesBuilderTest < Minitest::Test thread&.join end - # A tiny real-world bbox (a few blocks of central Bratislava) keeps the - # z0..15 tile_range small at every zoom level, so this stays fast even - # though it iterates all 16 levels. - def test_build_terrain_packs_fetched_tiles_and_converts + # A bbox nowhere near chunk-a/chunk-b — only planet.pmtiles matches, so + # this should be a single extract straight to the final path, no merge. + def test_build_terrain_extracts_only_planet_when_no_chunk_intersects ENV["PMTILES_BIN"] = @pmtiles_ok - thread = Maptool::PmtilesBuilder.build_terrain( - [48.140, 17.100, 48.141, 17.101], - data_dir: @data_dir, - fetch_tile: ->(_z, _x, _y) { "not a real png, write_terrarium_mbtiles just stores opaque bytes" }, - ) + thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { fake_manifest }) thread.join assert_path_exists File.join(@data_dir, "terrain.pmtiles") - status = Maptool::PmtilesBuilder.terrain_status - assert_equal "done", status[:state] - assert_operator status[:tiles_total], :>, 0 - assert_equal status[:tiles_total], status[:tiles_done] + assert_equal "done", Maptool::PmtilesBuilder.terrain_status[:state] + assert_equal ["extract https://example.com/planet.pmtiles"], log_lines end - # Every fetch_tile call blocks on a gate until explicitly told not to — - # with CONCURRENT_TILE_FETCHES workers pulling from a shared queue, that - # holds the whole pool at "in flight, none finished yet" deterministically - # (a big enough bbox that z0..15 yields far more tiles than worker slots, - # so most of the job is still queued up behind the blocked pool). - def test_build_terrain_reports_progress_while_running + # A bbox inside chunk-a's extent — planet.pmtiles (always) plus that one + # chunk get extracted, then merged. + def test_build_terrain_extracts_and_merges_a_single_intersecting_chunk ENV["PMTILES_BIN"] = @pmtiles_ok - concurrency = Maptool::PmtilesBuilder::CONCURRENT_TILE_FETCHES - gate = Queue.new - unblocked = false - fetch_tile = lambda do |_z, _x, _y| - gate.pop unless unblocked - "bytes" - end - thread = Maptool::PmtilesBuilder.build_terrain([47.5, 16.5, 48.5, 17.5], data_dir: @data_dir, fetch_tile: fetch_tile) + thread = Maptool::PmtilesBuilder.build_terrain([48.1, 17.1, 48.2, 17.2], data_dir: @data_dir, fetch_manifest: -> { fake_manifest }) + thread.join - # Let exactly one batch through (fills every worker's first call) then - # check before releasing the rest — done should be capped at the pool - # size, nowhere near the real total. - concurrency.times { gate.push(nil) } - sleep 0.05 - status = Maptool::PmtilesBuilder.terrain_status - assert_equal "running", status[:state] - assert_operator status[:tiles_total], :>, concurrency - assert_operator status[:tiles_done], :<, status[:tiles_total] - ensure - unblocked = true - concurrency.times { gate.push(nil) } # release whatever's still blocked in an in-progress gate.pop - thread&.join + assert_path_exists File.join(@data_dir, "terrain.pmtiles") + assert_equal "done", Maptool::PmtilesBuilder.terrain_status[:state] + extracted = log_lines.select { |line| line.start_with?("extract ") } + merged = log_lines.select { |line| line.start_with?("merge ") } + assert_equal ["extract https://example.com/planet.pmtiles", "extract https://example.com/chunk-a.pmtiles"], extracted + assert_equal 1, merged.size end - def test_build_terrain_still_succeeds_when_some_tiles_are_missing + # A bbox spanning both chunk-a and chunk-b — planet + both chunks, one + # 3-way merge. + def test_build_terrain_merges_multiple_intersecting_chunks ENV["PMTILES_BIN"] = @pmtiles_ok - thread = Maptool::PmtilesBuilder.build_terrain( - [48.140, 17.100, 48.141, 17.101], - data_dir: @data_dir, - fetch_tile: ->(_z, _x, _y) { nil }, # every tile "missing" — should skip, not fail, the same way HillshadeControl's own source tolerates gaps - ) + thread = Maptool::PmtilesBuilder.build_terrain([47.9, 16.9, 50.6, 20.6], data_dir: @data_dir, fetch_manifest: -> { fake_manifest }) thread.join assert_equal "done", Maptool::PmtilesBuilder.terrain_status[:state] + extracted = log_lines.count { |line| line.start_with?("extract ") } + assert_equal 3, extracted # planet + chunk-a + chunk-b + assert_equal 1, log_lines.count { |line| line.start_with?("merge ") } + end + + def test_build_terrain_reports_a_clear_error_when_the_manifest_is_missing_planet + ENV["PMTILES_BIN"] = @pmtiles_ok + thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { fake_manifest.reject { |i| i["name"] == "planet.pmtiles" } }) + thread.join + + status = Maptool::PmtilesBuilder.terrain_status + assert_equal "error", status[:state] + assert_match(/missing planet\.pmtiles/, status[:message]) + end + + def test_build_terrain_reports_an_error_status_on_non_zero_exit + ENV["PMTILES_BIN"] = @pmtiles_fail + thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { fake_manifest }) + thread.join + + refute_path_exists File.join(@data_dir, "terrain.pmtiles") + status = Maptool::PmtilesBuilder.terrain_status + assert_equal "error", status[:state] + assert_match(/exited with status/, status[:message]) + end + + def test_build_terrain_raises_when_one_is_already_running + ENV["PMTILES_BIN"] = @pmtiles_ok + gate = Queue.new + thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { gate.pop }) + + assert_raises(Maptool::PmtilesBuilder::BuildError) do + Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { "unused" }) + end + ensure + gate.push(fake_manifest) + thread&.join end end