From cd56ebe0a1fa0255a7568cf11784cd6d74733a53 Mon Sep 17 00:00:00 2001 From: Roman Priesol Date: Fri, 11 Sep 2026 01:25:08 +0200 Subject: [PATCH] Add in-app regeneration of offline.pmtiles and terrain.pmtiles A logged-in user can now trigger, from a new "Offline data" sidebar panel, what was previously a fully manual go-pmtiles workflow (docs/development.adoc): regenerate either file for the current map viewport, no area-picker (same "visible area is the area" convention the Overpass POI plugin already uses). - lib/maptool/pmtiles_builder.rb: two independent background jobs (Thread + Mutex-guarded status Hash, same shape as app.rb's own GNSS poller), each writing to a .tmp path and atomically renaming into place so a build never serves a partially-written file mid-run. offline.pmtiles is a single `pmtiles extract` against Protomaps' public daily basemap build (byte-range extraction, not a full ~120GB download). terrain.pmtiles has no equivalent public Terrarium-encoded archive to extract from, so it fetches individual Terrarium tiles itself, packs them into an intermediate MBTiles (new sqlite3 gem), then the same `pmtiles convert`. - 4 new app.rb routes (POST .../build, GET .../build-status) plus size/mtime added to the existing availability endpoints. - public/js/offline_data.js: new sidebar panel, polling build-status the same self-rescheduling-setTimeout shape livetrack.js already uses for its own refresh loop. - Both network-heavy pieces (the Protomaps build index, per-tile fetches) are injectable, so test/test_pmtiles_builder.rb exercises the actual build pipeline against a fake `pmtiles` binary without hitting a real network, matching this suite's existing keep-it-offline convention for external services. --- Gemfile | 1 + Gemfile.lock | 21 ++++ app.rb | 50 +++++++- docs/development.adoc | 56 +++++++-- lib/maptool/pmtiles_builder.rb | 217 +++++++++++++++++++++++++++++++++ public/css/style.css | 43 +++++++ public/js/icons.js | 10 ++ public/js/map.js | 2 + public/js/offline_data.js | 168 +++++++++++++++++++++++++ test/test_app.rb | 43 +++++++ test/test_helper.rb | 1 + test/test_pmtiles_builder.rb | 130 ++++++++++++++++++++ views/index.erb | 12 ++ views/layout.erb | 1 + 14 files changed, 740 insertions(+), 15 deletions(-) create mode 100644 lib/maptool/pmtiles_builder.rb create mode 100644 public/js/offline_data.js create mode 100644 test/test_pmtiles_builder.rb diff --git a/Gemfile b/Gemfile index c529183..99f2194 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,7 @@ gem "puma", "~> 8.0" gem "rexml", "~> 3.4" gem "oauth2", "~> 2.0" gem "rubyzip", "~> 2.4" +gem "sqlite3", "~> 2.0" group :development do gem "rerun" diff --git a/Gemfile.lock b/Gemfile.lock index db423d5..c4dfae9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -95,6 +95,16 @@ GEM snaky_hash (2.0.7) hashie (>= 0.1.0, < 6) version_gem (~> 1.1, >= 1.1.14) + sqlite3 (2.9.6-aarch64-linux-gnu) + sqlite3 (2.9.6-aarch64-linux-musl) + sqlite3 (2.9.6-arm-linux-gnu) + sqlite3 (2.9.6-arm-linux-musl) + sqlite3 (2.9.6-arm64-darwin) + sqlite3 (2.9.6-x86-linux-gnu) + sqlite3 (2.9.6-x86-linux-musl) + sqlite3 (2.9.6-x86_64-darwin) + sqlite3 (2.9.6-x86_64-linux-gnu) + sqlite3 (2.9.6-x86_64-linux-musl) tilt (2.9.0) uri (1.1.1) version_gem (1.1.15) @@ -122,6 +132,7 @@ DEPENDENCIES rubyzip (~> 2.4) sinatra (~> 4.2) sinatra-contrib (~> 4.2) + sqlite3 (~> 2.0) CHECKSUMS anonymous_loader (0.1.3) sha256=084a18e2439144d955447dc11dfc982f41fcd1583ad32d4d55151325dc44cb55 @@ -171,6 +182,16 @@ CHECKSUMS sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 sinatra-contrib (4.2.1) sha256=10d091c944d268aa910c618ea40a3c3ebe0533e6e32990d84af92235a3d26b4a snaky_hash (2.0.7) sha256=7d02c70012a3f932e48860cd024577908300c9aa615e0cb9b450aaa749cbcb4d + sqlite3 (2.9.6-aarch64-linux-gnu) sha256=d8b1f7d23efd7abac285775a9566562fc7debfef79d594e3a20354406fb7907c + sqlite3 (2.9.6-aarch64-linux-musl) sha256=3579e1c98cdc7ff5c3722847bb63ed4e1efb7ff675cb5e1e48ef2d4da5fb3bc9 + sqlite3 (2.9.6-arm-linux-gnu) sha256=33541500e3615da02afe54a9cc38b17a6985d3cf9d8b76d6d0a83002f114e7ec + sqlite3 (2.9.6-arm-linux-musl) sha256=c5490af48bb228fefa54314e9541375c3907e70f8109f3881b5ff97e1c93ae33 + sqlite3 (2.9.6-arm64-darwin) sha256=849b5d7f795e60fe25076d62c72dd722beb45b3850b516ad978d60ee848ec15b + sqlite3 (2.9.6-x86-linux-gnu) sha256=fbaa9f46f9708f57dd8a459b37fc269f9613e0cacf1547df01aa439cc45c20c0 + sqlite3 (2.9.6-x86-linux-musl) sha256=6715026fbb5530e810b28ef43b9c4f84cd3c991f67b9d808a31fcc32b847abbd + sqlite3 (2.9.6-x86_64-darwin) sha256=b5842fea77781c14da03fa7bc0feb82db03a69e135affcb6f5399cbd2797a5f3 + sqlite3 (2.9.6-x86_64-linux-gnu) sha256=613188ce02f614126ddbc38c5e217ccffd6306d0dcd9adca9764547aa890a634 + sqlite3 (2.9.6-x86_64-linux-musl) sha256=d493b11818a3573387a1d56e1ee8fa00da23a683a7a1cc063e7a0feeed843abf tilt (2.9.0) sha256=da5735d0280bba96e9a91041bb14aee435ccad5c17b0fa519249ae543d9aa3a5 uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 version_gem (1.1.15) sha256=a73241587b29252e3567e0c818ded80730eb754d43b508f6ce4db22ca9ba27d0 diff --git a/app.rb b/app.rb index 9bd7ab8..6cd5f42 100644 --- a/app.rb +++ b/app.rb @@ -12,6 +12,7 @@ require_relative "lib/maptool/livetrack_store" require_relative "lib/maptool/livetrack_client" require_relative "lib/maptool/kiwix" require_relative "lib/maptool/plugins" +require_relative "lib/maptool/pmtiles_builder" module Maptool class App < Sinatra::Base @@ -125,6 +126,12 @@ module Maptool halt 401, json(error: "login required", login_url: "/login") unless logged_in? end + def pmtiles_file_status(path) + return { available: false } unless File.exist?(path) + + { available: true, size: File.size(path), mtime: File.mtime(path).utc.iso8601 } + end + def callback_url "#{request.base_url}/auth/callback" end @@ -204,7 +211,7 @@ module Maptool end get "/api/offline-tiles" do - json available: File.exist?(OFFLINE_PMTILES_PATH) + json pmtiles_file_status(OFFLINE_PMTILES_PATH) end # Served as a plain range-request-capable file, not decoded server-side: @@ -216,11 +223,33 @@ module Maptool send_file OFFLINE_PMTILES_PATH, type: "application/octet-stream" end + # Regenerates offline.pmtiles for the given bbox (the current map + # viewport, same "visible area is the area" convention as the Overpass + # POI plugin) by extracting it from Protomaps' own daily full-planet + # basemap build — see lib/maptool/pmtiles_builder.rb. Runs in the + # background; poll build-status below for progress. + post "/api/offline-tiles/build" do + require_login! + bbox = %w[south west north east].map { |key| params[key].to_s } + halt 400, json(error: "south, west, north, east are required") if bbox.any?(&:empty?) + + begin + PmtilesBuilder.build_offline(bbox.map(&:to_f), data_dir: DATA_DIR) + rescue PmtilesBuilder::BuildError => e + halt 409, json(error: e.message) + end + json status: "started" + end + + get "/api/offline-tiles/build-status" do + json PmtilesBuilder.offline_status + end + # Same optional-local-file story as offline.pmtiles above — see # docs/development.adoc for how to generate one, until then hillshade # just stays unavailable while the offline basemap is active. get "/api/terrain-tiles" do - json available: File.exist?(TERRAIN_PMTILES_PATH) + json pmtiles_file_status(TERRAIN_PMTILES_PATH) end get "/tiles/terrain.pmtiles" do @@ -229,6 +258,23 @@ module Maptool send_file TERRAIN_PMTILES_PATH, type: "application/octet-stream" end + post "/api/terrain-tiles/build" do + require_login! + bbox = %w[south west north east].map { |key| params[key].to_s } + halt 400, json(error: "south, west, north, east are required") if bbox.any?(&:empty?) + + begin + PmtilesBuilder.build_terrain(bbox.map(&:to_f), data_dir: DATA_DIR) + rescue PmtilesBuilder::BuildError => e + halt 409, json(error: e.message) + end + json status: "started" + end + + get "/api/terrain-tiles/build-status" do + json PmtilesBuilder.terrain_status + end + # Pinged once per page load (public/js/map.js) — url is a same-origin # path (not kiwix-serve's own address, which the browser can't reach # directly behind the reverse proxy in front of this app), proxied by diff --git a/docs/development.adoc b/docs/development.adoc index 0cce540..ed849dd 100644 --- a/docs/development.adoc +++ b/docs/development.adoc @@ -27,18 +27,25 @@ reload (e.g. Ctrl/Cmd+Shift+R) if that happens. To enable the offline style, place a PMTiles file at `data/offline.pmtiles` (see `docs/architecture.adoc` for how it is -served and styled). It is not part of the repository — obtain or -build one locally: +served and styled). Logged-in users can generate this from the app +itself — see "In-app regeneration" below — or it can be built by hand: -* The simplest source is a pre-built extract from - https://app.protomaps.com[protomaps.com's extract builder] — pick a - region and download it as `.pmtiles`. Its schema is what - `protomaps-themes-base` (the styling library used for the offline - style) expects. +* Extract a region from Protomaps' own daily full-planet basemap build + (https://build-metadata.protomaps.dev/builds.json[build index], + files at `https://build.protomaps.com/`, ~120GB, ODbL/OSM + attribution) with the + https://github.com/protomaps/go-pmtiles[go-pmtiles] CLI's `extract` + subcommand, which pulls only the byte ranges covering a bbox over + HTTP range requests rather than the whole file: + `pmtiles extract https://build.protomaps.com/ data/offline.pmtiles --bbox=west,south,east,north --maxzoom=15`. + (The old `app.protomaps.com` extract-builder UI this section used to + point at has since redirected elsewhere — this is the same + underlying mechanism, just the CLI instead of a web UI.) * To convert some other vector tile source (e.g. an `.mbtiles` file) - into PMTiles, use the - https://github.com/protomaps/go-pmtiles[go-pmtiles] CLI: - `pmtiles convert in.mbtiles data/offline.pmtiles`. + into PMTiles instead, use the same CLI's `convert` subcommand: + `pmtiles convert in.mbtiles data/offline.pmtiles`. Its schema is + what `protomaps-themes-base` (the styling library used for the + offline style) expects. Once the file is in place, `GET /api/offline-tiles` reports `available: true` and the map defaults to the offline style. @@ -50,15 +57,38 @@ The sidebar's "Terrain shading" toggle works online out of the box no API key). For it to work offline too, place a PMTiles archive of Terrarium-encoded `raster-dem` tiles at `data/terrain.pmtiles` (`GET /api/terrain-tiles` reports `available: true` once it's there, -same pattern as offline map data above). It is not part of the -repository — build one locally from a DEM source (e.g. Copernicus -GLO-30 or SRTM) with a tool like +same pattern as offline map data above). Also generatable in-app (see +below) or by hand — build one locally from a DEM source (e.g. +Copernicus GLO-30 or SRTM) with a tool like https://github.com/mapbox/rio-rgbify[rio-rgbify] (GeoTIFF -> Terrarium-encoded PNG tiles in an `.mbtiles`), then `pmtiles convert out.mbtiles data/terrain.pmtiles` with https://github.com/protomaps/go-pmtiles[go-pmtiles], same as `data/offline.pmtiles`. +== In-app regeneration + +A logged-in user can also generate both files above from the app +itself, no local tooling needed — the sidebar's "Offline data" icon +(Offline hub group) shows each file's status (size, last updated) and +a "Regenerate for current view" button. The region is always the +current map viewport, same "no area-picker, the visible area is the +area" convention the Overpass POI plugin already uses below. This +wraps exactly the manual `go-pmtiles` workflows documented above +(`lib/maptool/pmtiles_builder.rb`) — the server needs the +https://github.com/protomaps/go-pmtiles[go-pmtiles] CLI installed and +on `PATH` (override with the `PMTILES_BIN` env var if it lives +somewhere else); regenerating terrain.pmtiles this way fetches +individual Terrarium tiles itself and packs them into an MBTiles file +(the `sqlite3` gem) before the same `pmtiles convert` step, since no +public Terrarium-encoded PMTiles archive exists to `extract` from the +way offline.pmtiles above can. + +A build runs in the background (`POST /api/offline-tiles/build` / +`/api/terrain-tiles/build`, polled via the matching `*/build-status` +route) — only one of each kind at a time; starting a second while one's +running gets a 409. + == Camper-relevant POIs (Overpass) `rake overpass:pois[south,west,north,east,out.geojson]` queries diff --git a/lib/maptool/pmtiles_builder.rb b/lib/maptool/pmtiles_builder.rb new file mode 100644 index 0000000..b2090d8 --- /dev/null +++ b/lib/maptool/pmtiles_builder.rb @@ -0,0 +1,217 @@ +require "open3" +require "net/http" +require "uri" +require "json" +require "sqlite3" +require "tmpdir" + +module Maptool + # Regenerates data/offline.pmtiles and data/terrain.pmtiles for a given + # bbox by shelling out to go-pmtiles — the same tool docs/development.adoc + # already documents running by hand. No job queue: each build runs on its + # own background Thread and reports into a Mutex-guarded status Hash a + # route can poll, the same shape app.rb's own GNSS poller (GNSS_LATEST/ + # GNSS_LATEST_MUTEX) already uses. + module PmtilesBuilder + class BuildError < StandardError; end + + # Protomaps' own daily full-planet basemap build index — extract_offline + # below only pulls the HTTP byte ranges covering the requested bbox + # (go-pmtiles's `extract` supports a remote source over range requests), + # never the whole ~120GB file. + PROTOMAPS_BUILDS_INDEX = "https://build-metadata.protomaps.dev/builds.json" + PROTOMAPS_BUILD_BASE = "https://build.protomaps.com" + + # Same public, no-API-key Terrarium tile source public/js/overlays.js's + # ONLINE_TERRAIN_TILES already fetches client-side — no shared constant + # between Ruby and JS, just the same literal URL template. Matches that + # source's own online maxzoom. + TERRARIUM_URL_TEMPLATE = "https://s3.amazonaws.com/elevation-tiles-prod/terrarium/%d/%d/%d.png" + TERRAIN_MAX_ZOOM = 15 + + OFFLINE_STATUS = { state: "idle" } + TERRAIN_STATUS = { state: "idle" } + OFFLINE_MUTEX = Mutex.new + TERRAIN_MUTEX = Mutex.new + + def self.offline_status + OFFLINE_MUTEX.synchronize { OFFLINE_STATUS.dup } + end + + def self.terrain_status + TERRAIN_MUTEX.synchronize { TERRAIN_STATUS.dup } + end + + # Test-only: the status hashes are process-wide state (same caveat as + # GPX_STORE/LIVETRACK_STORE in app.rb), so the suite needs a way to + # reset them between tests the way AppTestHelper#teardown already does + # for those. + def self.reset! + OFFLINE_MUTEX.synchronize { OFFLINE_STATUS.replace(state: "idle") } + TERRAIN_MUTEX.synchronize { TERRAIN_STATUS.replace(state: "idle") } + end + + # bounds: [south, west, north, east]. Returns once the background Thread + # has started, not once the build finishes — raises BuildError (the + # caller turns this into a 409) if one's already running. + # + # fetch_build_url exists as an injectable seam so tests can exercise the + # rest of this (temp path, go-pmtiles invocation, atomic rename) without + # a real network call — same reasoning test_helper.rb already gives for + # keeping /api/plugins route tests off the real Overpass API. + def self.build_offline(bounds, data_dir:, fetch_build_url: method(:latest_protomaps_build_url)) + begin_run!(OFFLINE_MUTEX, OFFLINE_STATUS) + Thread.new { run_job(OFFLINE_MUTEX, OFFLINE_STATUS) { extract_offline(bounds, data_dir, fetch_build_url) } } + end + + # fetch_tile: same injectable-seam reasoning as fetch_build_url above, + # for the many individual Terrarium tile fetches this needs instead of + # one index lookup. + def self.build_terrain(bounds, data_dir:, fetch_tile: method(:fetch_terrarium_tile)) + begin_run!(TERRAIN_MUTEX, TERRAIN_STATUS) + Thread.new { run_job(TERRAIN_MUTEX, TERRAIN_STATUS) { build_terrain_pmtiles(bounds, data_dir, fetch_tile) } } + end + + def self.begin_run!(mutex, status) + mutex.synchronize do + raise BuildError, "a build is already running" if status[:state] == "running" + status.replace(state: "running", started_at: Time.now.utc.iso8601) + end + end + private_class_method :begin_run! + + def self.run_job(mutex, status) + yield + mutex.synchronize { status.merge!(state: "done", finished_at: Time.now.utc.iso8601).delete(:message) } + rescue => e + mutex.synchronize { status.merge!(state: "error", finished_at: Time.now.utc.iso8601, message: e.message) } + end + private_class_method :run_job + + # --- offline.pmtiles --- + + def self.extract_offline(bounds, data_dir, fetch_build_url) + source = fetch_build_url.call + south, west, north, east = bounds + tmp_path = File.join(data_dir, "offline.pmtiles.tmp") + + run_pmtiles!("extract", source, tmp_path, "--bbox=#{west},#{south},#{east},#{north}", "--maxzoom=15") + File.rename(tmp_path, File.join(data_dir, "offline.pmtiles")) + ensure + File.delete(tmp_path) if tmp_path && File.exist?(tmp_path) + end + private_class_method :extract_offline + + def self.latest_protomaps_build_url + response = Net::HTTP.get_response(URI(PROTOMAPS_BUILDS_INDEX)) + raise BuildError, "could not reach #{PROTOMAPS_BUILDS_INDEX}" unless response.is_a?(Net::HTTPSuccess) + + builds = JSON.parse(response.body) + raise BuildError, "protomaps build index was empty" if builds.empty? + + "#{PROTOMAPS_BUILD_BASE}/#{builds.last.fetch("key")}" + rescue JSON::ParserError + raise BuildError, "protomaps build index was not valid JSON" + end + private_class_method :latest_protomaps_build_url + + # --- terrain.pmtiles --- + + def self.build_terrain_pmtiles(bounds, data_dir, fetch_tile) + south, west, north, east = bounds + tmp_mbtiles = File.join(Dir.tmpdir, "maptool-terrain-#{Process.pid}-#{Time.now.to_i}.mbtiles") + tmp_path = File.join(data_dir, "terrain.pmtiles.tmp") + + write_terrarium_mbtiles(tmp_mbtiles, south, west, north, east, fetch_tile) + run_pmtiles!("convert", tmp_mbtiles, tmp_path) + File.rename(tmp_path, File.join(data_dir, "terrain.pmtiles")) + ensure + File.delete(tmp_mbtiles) if tmp_mbtiles && File.exist?(tmp_mbtiles) + File.delete(tmp_path) if tmp_path && File.exist?(tmp_path) + end + private_class_method :build_terrain_pmtiles + + # Terrarium PNG tiles for z0..TERRAIN_MAX_ZOOM over the bbox, packed + # into a minimal MBTiles (SQLite) archive — go-pmtiles' own `convert` + # only accepts MBTiles/PMTiles as input, no raw tile directory, and no + # public Terrarium-encoded PMTiles archive is known to exist to + # `extract` from the way offline.pmtiles does above. + # + # ponytail: fetches tiles one at a time, serially — fine for a + # park-sized region, slow for a big one. Add a small bounded-concurrency + # fetch pool if regenerating a large area turns out to matter in + # practice. + def self.write_terrarium_mbtiles(path, south, west, north, east, fetch_tile) + db = SQLite3::Database.new(path) + db.execute_batch(<<~SQL) + CREATE TABLE metadata (name TEXT, value TEXT); + CREATE TABLE tiles (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_data BLOB); + CREATE UNIQUE INDEX tile_index ON tiles (zoom_level, tile_column, tile_row); + SQL + { "format" => "png", "name" => "terrain", "type" => "baselayer", "minzoom" => "0", "maxzoom" => TERRAIN_MAX_ZOOM.to_s }.each do |name, value| + db.execute("INSERT INTO metadata (name, value) VALUES (?, ?)", [name, value]) + end + + (0..TERRAIN_MAX_ZOOM).each do |z| + tile_range(z, south, west, north, east).each do |x, y| + bytes = fetch_tile.call(z, x, y) + next unless bytes + + # MBTiles uses TMS row numbering (0 at the south) — the opposite + # of the XYZ scheme both the Terrarium source and this app's own + # online raster-dem layer use (0 at the north) — flipped here so + # go-pmtiles ends up serving the same tile at the same z/x/y this + # app already requests online. + tms_y = (2**z) - 1 - y + db.execute("INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", [z, x, tms_y, bytes.b]) + end + end + ensure + db&.close + end + private_class_method :write_terrarium_mbtiles + + def self.tile_range(z, south, west, north, east) + n = 2**z + x_min, x_max = [lng_to_tile_x(west, n), lng_to_tile_x(east, n)].minmax + y_min, y_max = [lat_to_tile_y(north, n), lat_to_tile_y(south, n)].minmax + x_min.upto(x_max).flat_map { |x| y_min.upto(y_max).map { |y| [x, y] } } + end + private_class_method :tile_range + + def self.lng_to_tile_x(lng, n) + (((lng + 180) / 360) * n).floor.clamp(0, n - 1) + end + private_class_method :lng_to_tile_x + + def self.lat_to_tile_y(lat, n) + lat_rad = lat * Math::PI / 180 + (((1 - (Math.log(Math.tan(lat_rad) + (1 / Math.cos(lat_rad))) / Math::PI)) / 2) * n).floor.clamp(0, n - 1) + end + private_class_method :lat_to_tile_y + + def self.fetch_terrarium_tile(z, x, y) + response = Net::HTTP.get_response(URI(format(TERRARIUM_URL_TEMPLATE, z: z, x: x, y: y))) + response.is_a?(Net::HTTPSuccess) ? response.body : nil + rescue StandardError + nil + end + private_class_method :fetch_terrarium_tile + + # --- shared --- + + # ENV read lazily (not a load-time constant) so tests can point this at + # a fake script without a require-order dance. + def self.pmtiles_bin + ENV.fetch("PMTILES_BIN", "pmtiles") + end + + def self.run_pmtiles!(*args) + _stdout, stderr, status = Open3.capture3(pmtiles_bin, *args) + raise BuildError, "pmtiles exited with status #{status.exitstatus}: #{stderr.strip}" unless status.success? + rescue Errno::ENOENT, Errno::EACCES => e + raise BuildError, "could not run '#{pmtiles_bin}' (#{e.message}) — is go-pmtiles installed and on PATH? See docs/development.adoc." + end + private_class_method :run_pmtiles! + end +end diff --git a/public/css/style.css b/public/css/style.css index 0753c90..12979ff 100644 --- a/public/css/style.css +++ b/public/css/style.css @@ -397,6 +397,49 @@ html, body { visibility: visible; } +.offline-data-panel__body { + padding: 8px 12px; +} + +.offline-data-panel__row + .offline-data-panel__row { + margin-top: 10px; + padding-top: 10px; + border-top: 1px solid #eee; +} + +.offline-data-panel__row-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; +} + +.offline-data-panel__build { + flex-shrink: 0; + border: 1px solid #ccc; + border-radius: 4px; + background: #fff; + padding: 4px 8px; + font: inherit; + font-size: 12px; + cursor: pointer; +} + +.offline-data-panel__build:hover { + background: #f0f0f0; +} + +.offline-data-panel__build:disabled { + opacity: 0.5; + cursor: default; +} + +.offline-data-panel__status { + margin: 6px 0 0; + color: #666; + font-size: 12px; +} + .gpx-panel { position: fixed; top: 0; diff --git a/public/js/icons.js b/public/js/icons.js index ab3fa44..545f630 100644 --- a/public/js/icons.js +++ b/public/js/icons.js @@ -416,6 +416,16 @@ Object.assign(POI_ICONS, { shop_convenience: STORE_ICON, }); +// Hand-drawn, not NPS-sourced — no "download offline map data" pictogram +// in that set either. A tray with a downward arrow into it, same solid +// 22x22 currentColor convention as CONTOUR_ICON/TERRAIN_SHADOW_ICON above +// — used for the "Offline data" sidebar icon (offline_data.js). +export const OFFLINE_DATA_ICON = ` + + + +`; + // src/bookstore-22.svg (NPS symbol library, same source/license as // POI_ICONS above) — used as the fixed marker glyph for the "Wikipedia" // map layer (gpx.js), not part of POI_ICONS itself: that set is keyed by diff --git a/public/js/map.js b/public/js/map.js index 58974e7..40ad1d4 100644 --- a/public/js/map.js +++ b/public/js/map.js @@ -15,6 +15,7 @@ import { NotesControl } from "/js/notes.js"; import { PluginsControl } from "/js/plugins.js"; import { LocateControl } from "/js/locate.js"; import { SearchControl } from "/js/search.js"; +import { OfflineDataControl } from "/js/offline_data.js"; import { LAYERS_ICON, WAYPOINT_ICON, SUNRISE_ICON, SUNSET_ICON, PENDING_ICON } from "/js/icons.js"; import { formatCoords, formatSunInfo } from "/js/sun.js"; import { notifySidePanelOpen, onlyOneSidePanelOpen } from "/js/panels.js"; @@ -515,6 +516,7 @@ if (isLoggedIn()) new PluginsControl(map).mount(sidebar); sidebar.insertAdjacentHTML("beforeend", `
`); layerToggle.mount(sidebar); layerToggle.setActive("online"); +new OfflineDataControl(map).mount(sidebar); sidebar.insertAdjacentHTML("beforeend", `
`); gnssControl.mount(sidebar); diff --git a/public/js/offline_data.js b/public/js/offline_data.js new file mode 100644 index 0000000..ab3cfb5 --- /dev/null +++ b/public/js/offline_data.js @@ -0,0 +1,168 @@ +import { OFFLINE_DATA_ICON } from "/js/icons.js"; +import { notifySidePanelOpen, onlyOneSidePanelOpen } from "/js/panels.js"; +import { isLoggedIn } from "/js/auth.js"; + +const POLL_INTERVAL_MS = 3000; + +function formatBytes(bytes) { + const mb = bytes / (1024 * 1024); + return mb >= 1024 ? `${(mb / 1024).toFixed(1)} GB` : `${mb.toFixed(1)} MB`; +} + +// One row of the Offline data panel — status text plus a "Regenerate for +// current view" button, for either offline.pmtiles or terrain.pmtiles. +// Both files behave identically from here (same status/build/build-status +// endpoint shape, see lib/maptool/pmtiles_builder.rb), just against +// different URLs/labels, so one class covers both rather than writing the +// same fetch/poll logic twice. +class DataFileRow { + constructor(map, { label, statusUrl, buildUrl, buildStatusUrl }) { + this._map = map; + this._label = label; + this._statusUrl = statusUrl; + this._buildUrl = buildUrl; + this._buildStatusUrl = buildStatusUrl; + this._exists = false; + } + + mount(container) { + const row = document.createElement("div"); + row.className = "offline-data-panel__row"; + row.innerHTML = ` +
+ ${this._label} + +
+

Checking…

+ `; + this._buildButton = row.querySelector(".offline-data-panel__build"); + this._statusEl = row.querySelector(".offline-data-panel__status"); + // Viewing status works for anyone; only triggering a build needs a + // login — same finer-grained gating measure.js/gpx.js use for their + // own write-only affordances, rather than hiding this whole control. + this._buildButton.hidden = !isLoggedIn(); + this._buildButton.addEventListener("click", () => this._build()); + + container.appendChild(row); + this._refreshStatus(); + } + + async _refreshStatus() { + let data; + try { + data = await (await fetch(this._statusUrl)).json(); + } catch { + this._statusEl.textContent = "Could not check status."; + return; + } + this._exists = data.available; + this._statusEl.textContent = data.available + ? `${formatBytes(data.size)} · updated ${new Date(data.mtime).toLocaleString([], { dateStyle: "short", timeStyle: "short" })}` + : "Not generated yet."; + } + + async _build() { + if (this._exists && !window.confirm(`Replace the existing ${this._label.toLowerCase()} with a fresh build for the current view?`)) { + return; + } + + const bounds = this._map.getBounds(); + const body = new URLSearchParams({ + south: bounds.getSouth(), + west: bounds.getWest(), + north: bounds.getNorth(), + east: bounds.getEast(), + }); + + this._buildButton.disabled = true; + this._statusEl.textContent = "Starting…"; + try { + const response = await fetch(this._buildUrl, { method: "POST", body }); + const data = await response.json().catch(() => ({})); + if (!response.ok) throw new Error(data.error || `build failed: ${response.status}`); + } catch (err) { + this._statusEl.textContent = err.message; + this._buildButton.disabled = false; + return; + } + + this._statusEl.textContent = "Building…"; + this._poll(); + } + + // Self-rescheduling setTimeout loop, same shape as livetrack.js's + // _scheduleRefresh — check status, only reschedule while still running. + _poll() { + setTimeout(async () => { + let status; + try { + status = await (await fetch(this._buildStatusUrl)).json(); + } catch { + this._poll(); // transient failure — try again next tick + return; + } + + if (status.state === "running") { + this._poll(); + return; + } + + this._buildButton.disabled = false; + if (status.state === "error") { + this._statusEl.textContent = `Build failed: ${status.message}`; + } else { + this._refreshStatus(); + } + }, POLL_INTERVAL_MS); + } +} + +// A sidebar panel wrapping the manual go-pmtiles workflow +// docs/development.adoc documents — "Regenerate for current view" on +// either file shells out to it server-side instead of an admin running it +// by hand (see lib/maptool/pmtiles_builder.rb). No area-picker: the +// current map viewport is the region, same convention the Overpass POI +// plugin already uses for "give me a bbox." +export class OfflineDataControl { + constructor(map) { + this._rows = [ + new DataFileRow(map, { + label: "Offline map data", + statusUrl: "/api/offline-tiles", + buildUrl: "/api/offline-tiles/build", + buildStatusUrl: "/api/offline-tiles/build-status", + }), + new DataFileRow(map, { + label: "Offline terrain data", + statusUrl: "/api/terrain-tiles", + buildUrl: "/api/terrain-tiles/build", + buildStatusUrl: "/api/terrain-tiles/build-status", + }), + ]; + } + + mount(sidebar) { + this._container = document.createElement("div"); + this._container.className = "sidebar-item"; + this._container.innerHTML = ``; + this._icon = this._container.querySelector(".sidebar-icon"); + this._icon.addEventListener("click", () => this._togglePanel()); + + this._panel = document.getElementById("offline-data-panel"); + this._panel.querySelector(".basemap-panel__close").addEventListener("click", () => this._togglePanel(false)); + + const body = this._panel.querySelector(".offline-data-panel__body"); + this._rows.forEach((row) => row.mount(body)); + + onlyOneSidePanelOpen(this._panel, () => this._icon.classList.remove("active")); + + sidebar.appendChild(this._container); + } + + _togglePanel(force) { + const open = force === undefined ? this._panel.hidden : force; + this._panel.hidden = !open; + this._icon.classList.toggle("active", open); + if (open) notifySidePanelOpen(this._panel); + } +} diff --git a/test/test_app.rb b/test/test_app.rb index cf0d64e..d26324e 100644 --- a/test/test_app.rb +++ b/test/test_app.rb @@ -70,6 +70,8 @@ class AppTest < Minitest::Test assert_equal 401, request.delete("/api/gpx/folder/x").status assert_equal 401, request.post("/api/gpx/move").status assert_equal 401, request.post("/api/plugins/test-plugin/run").status + assert_equal 401, request.post("/api/offline-tiles/build").status + assert_equal 401, request.post("/api/terrain-tiles/build").status end # --- GPX/FIT upload and the tree --- @@ -347,6 +349,47 @@ class AppTest < Minitest::Test assert_equal 502, response.status end + # --- pmtiles regeneration --- + # + # The full build (network + go-pmtiles) is covered in isolation, with + # fakes injected, by test_pmtiles_builder.rb — same "keep this offline" + # reasoning as the rest of this file. What's route-specific and worth + # covering here is the bbox validation and the 409-while-running guard, + # neither of which needs a real build to actually run. + + def test_offline_tiles_build_status_defaults_to_idle + assert_equal({ "state" => "idle" }, json_body(request.get("/api/offline-tiles/build-status"))) + assert_equal({ "state" => "idle" }, json_body(request.get("/api/terrain-tiles/build-status"))) + end + + def test_offline_tiles_build_requires_a_bbox + response = request.post("/api/offline-tiles/build", **logged_in_env) + assert_equal 400, response.status + end + + def test_terrain_tiles_build_requires_a_bbox + response = request.post("/api/terrain-tiles/build", **logged_in_env) + assert_equal 400, response.status + end + + # fetch_build_url blocks on a Queue, called directly (not through the + # route, which has no way to inject a fake) so the build is + # deterministically still "running" when the route call below happens. + def test_offline_tiles_build_is_409_when_one_is_already_running + gate = Queue.new + thread = Maptool::PmtilesBuilder.build_offline([1, 2, 3, 4], data_dir: Maptool::App::DATA_DIR, fetch_build_url: -> { gate.pop }) + + response = request.post( + "/api/offline-tiles/build", + params: { south: "1", west: "2", north: "3", east: "4" }, + **logged_in_env, + ) + assert_equal 409, response.status + ensure + gate.push("unused") + thread&.join + end + # --- LiveTrack webhook --- def test_livetrack_webhook_requires_the_configured_secret diff --git a/test/test_helper.rb b/test/test_helper.rb index c5c319e..b266ff8 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -66,6 +66,7 @@ module AppTestHelper def teardown FileUtils.rm_rf(Dir.glob(File.join(Maptool::App::DATA_DIR, "gpx", "*"))) File.write(File.join(Maptool::App::DATA_DIR, "livetrack.json"), "[]") + Maptool::PmtilesBuilder.reset! super end diff --git a/test/test_pmtiles_builder.rb b/test/test_pmtiles_builder.rb new file mode 100644 index 0000000..5cde21f --- /dev/null +++ b/test/test_pmtiles_builder.rb @@ -0,0 +1,130 @@ +require_relative "test_helper" +require "maptool/pmtiles_builder" +require "tmpdir" +require "fileutils" + +# Network calls (the protomaps build index, individual Terrarium tile +# fetches) are never exercised here — same "keep this offline/deterministic" +# reasoning test_helper.rb already gives for /api/plugins staying off the +# real Overpass API — build_offline/build_terrain take fetch_build_url/ +# fetch_tile precisely so tests can swap those out. A fake `pmtiles` CLI +# (PMTILES_BIN) stands in for the real go-pmtiles binary. +class PmtilesBuilderTest < Minitest::Test + def setup + @data_dir = Dir.mktmpdir("maptool-pmtiles-test-data") + @bin_dir = Dir.mktmpdir("maptool-pmtiles-test-bin") + + # $3 is the output path either way: `extract SOURCE OUTPUT --bbox=... + # --maxzoom=...` and `convert INPUT OUTPUT` both put it 3rd, even though + # it's not the *last* arg in the extract case (the --flags follow it). + @pmtiles_ok = File.join(@bin_dir, "pmtiles-ok") + File.write(@pmtiles_ok, "#!/bin/sh\necho fake > \"$3\"\n") + File.chmod(0o755, @pmtiles_ok) + + @pmtiles_fail = File.join(@bin_dir, "pmtiles-fail") + File.write(@pmtiles_fail, "#!/bin/sh\necho 'boom' >&2\nexit 1\n") + File.chmod(0o755, @pmtiles_fail) + + @original_pmtiles_bin = ENV["PMTILES_BIN"] + Maptool::PmtilesBuilder.reset! + end + + def teardown + FileUtils.remove_entry(@data_dir) + FileUtils.remove_entry(@bin_dir) + ENV["PMTILES_BIN"] = @original_pmtiles_bin + Maptool::PmtilesBuilder.reset! + end + + def test_offline_and_terrain_status_start_idle + assert_equal({ state: "idle" }, Maptool::PmtilesBuilder.offline_status) + assert_equal({ state: "idle" }, Maptool::PmtilesBuilder.terrain_status) + end + + def test_build_offline_writes_the_file_and_reports_done + ENV["PMTILES_BIN"] = @pmtiles_ok + thread = Maptool::PmtilesBuilder.build_offline( + [1, 2, 3, 4], + data_dir: @data_dir, + fetch_build_url: -> { "https://example.com/fake.pmtiles" }, + ) + thread.join + + assert_path_exists File.join(@data_dir, "offline.pmtiles") + refute_path_exists File.join(@data_dir, "offline.pmtiles.tmp") + assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state] + end + + def test_build_offline_reports_an_error_status_on_non_zero_exit + ENV["PMTILES_BIN"] = @pmtiles_fail + thread = Maptool::PmtilesBuilder.build_offline( + [1, 2, 3, 4], + data_dir: @data_dir, + fetch_build_url: -> { "https://example.com/fake.pmtiles" }, + ) + thread.join + + refute_path_exists File.join(@data_dir, "offline.pmtiles") + status = Maptool::PmtilesBuilder.offline_status + assert_equal "error", status[:state] + assert_match(/exited with status/, status[:message]) + end + + def test_build_offline_reports_a_clear_error_when_the_binary_is_missing + ENV["PMTILES_BIN"] = File.join(@bin_dir, "does-not-exist") + thread = Maptool::PmtilesBuilder.build_offline( + [1, 2, 3, 4], + data_dir: @data_dir, + fetch_build_url: -> { "https://example.com/fake.pmtiles" }, + ) + thread.join + + status = Maptool::PmtilesBuilder.offline_status + assert_equal "error", status[:state] + assert_match(/go-pmtiles installed/, status[:message]) + end + + # fetch_build_url blocks on a Queue so the build is deterministically + # still "running" when the second call happens, rather than racing a real + # one to see which state wins. + def test_build_offline_raises_when_one_is_already_running + ENV["PMTILES_BIN"] = @pmtiles_ok + gate = Queue.new + thread = Maptool::PmtilesBuilder.build_offline([1, 2, 3, 4], data_dir: @data_dir, fetch_build_url: -> { gate.pop }) + + assert_raises(Maptool::PmtilesBuilder::BuildError) do + Maptool::PmtilesBuilder.build_offline([1, 2, 3, 4], data_dir: @data_dir, fetch_build_url: -> { "unused" }) + end + ensure + gate.push("https://example.com/fake.pmtiles") + thread&.join + end + + # A tiny real-world bbox (a few blocks of central Bratislava) keeps the + # z0..15 tile_range small at every zoom level, so this stays fast even + # though it iterates all 16 levels. + def test_build_terrain_packs_fetched_tiles_and_converts + ENV["PMTILES_BIN"] = @pmtiles_ok + thread = Maptool::PmtilesBuilder.build_terrain( + [48.140, 17.100, 48.141, 17.101], + data_dir: @data_dir, + fetch_tile: ->(_z, _x, _y) { "not a real png, write_terrarium_mbtiles just stores opaque bytes" }, + ) + thread.join + + assert_path_exists File.join(@data_dir, "terrain.pmtiles") + assert_equal "done", Maptool::PmtilesBuilder.terrain_status[:state] + end + + def test_build_terrain_still_succeeds_when_some_tiles_are_missing + ENV["PMTILES_BIN"] = @pmtiles_ok + thread = Maptool::PmtilesBuilder.build_terrain( + [48.140, 17.100, 48.141, 17.101], + data_dir: @data_dir, + fetch_tile: ->(_z, _x, _y) { nil }, # every tile "missing" — should skip, not fail, the same way HillshadeControl's own source tolerates gaps + ) + thread.join + + assert_equal "done", Maptool::PmtilesBuilder.terrain_status[:state] + end +end diff --git a/views/index.erb b/views/index.erb index 08208fb..2d9a5b7 100644 --- a/views/index.erb +++ b/views/index.erb @@ -88,3 +88,15 @@
    + + diff --git a/views/layout.erb b/views/layout.erb index 8166e3c..8a9304e 100644 --- a/views/layout.erb +++ b/views/layout.erb @@ -41,6 +41,7 @@ "/js/locate.js": "/js/locate.js?v=<%= ASSET_VERSION %>", "/js/measure.js": "/js/measure.js?v=<%= ASSET_VERSION %>", "/js/notes.js": "/js/notes.js?v=<%= ASSET_VERSION %>", + "/js/offline_data.js": "/js/offline_data.js?v=<%= ASSET_VERSION %>", "/js/overlays.js": "/js/overlays.js?v=<%= ASSET_VERSION %>", "/js/panels.js": "/js/panels.js?v=<%= ASSET_VERSION %>", "/js/plugins.js": "/js/plugins.js?v=<%= ASSET_VERSION %>",