Show build progress instead of a static "Building…"
CI / check (push) Successful in 18s
CI / deploy (push) Successful in 13s

terrain.pmtiles can take minutes (tiles fetched one at a time) with no
way to tell a slow build from a stuck one. Now reports tiles_done/
tiles_total into the same status Hash the frontend already polls, and
the panel shows elapsed time (both files) plus tile counts (terrain
only) so the numbers visibly move every ~3s poll instead of sitting on
one static string.
This commit is contained in:
randy committed 2026-09-11 11:57:43 +02:00
1 parent d72f8a3c94
commit bbe9f38bda
3 files changed
+68 -8

No files matched your search

+21 -7
View File
@@ -66,10 +66,14 @@ module Maptool
# fetch_tile: same injectable-seam reasoning as fetch_build_url above, # fetch_tile: same injectable-seam reasoning as fetch_build_url above,
# for the many individual Terrarium tile fetches this needs instead of # for the many individual Terrarium tile fetches this needs instead of
# one index lookup. # one index lookup. This build can run for minutes with nothing else to
# show for it, so write_terrarium_mbtiles reports tiles_done/tiles_total
# into the status Hash as it goes — otherwise a poller sees "running"
# the whole time with no way to tell it apart from stuck.
def self.build_terrain(bounds, data_dir:, fetch_tile: method(:fetch_terrarium_tile)) def self.build_terrain(bounds, data_dir:, fetch_tile: method(:fetch_terrarium_tile))
begin_run!(TERRAIN_MUTEX, TERRAIN_STATUS) begin_run!(TERRAIN_MUTEX, TERRAIN_STATUS)
Thread.new { run_job(TERRAIN_MUTEX, TERRAIN_STATUS) { build_terrain_pmtiles(bounds, data_dir, fetch_tile) } } progress = ->(done, total) { TERRAIN_MUTEX.synchronize { TERRAIN_STATUS.merge!(tiles_done: done, tiles_total: total) } }
Thread.new { run_job(TERRAIN_MUTEX, TERRAIN_STATUS) { build_terrain_pmtiles(bounds, data_dir, fetch_tile, progress) } }
end end
def self.begin_run!(mutex, status) def self.begin_run!(mutex, status)
@@ -117,12 +121,12 @@ module Maptool
# --- terrain.pmtiles --- # --- terrain.pmtiles ---
def self.build_terrain_pmtiles(bounds, data_dir, fetch_tile) def self.build_terrain_pmtiles(bounds, data_dir, fetch_tile, progress)
south, west, north, east = bounds south, west, north, east = bounds
tmp_mbtiles = File.join(Dir.tmpdir, "maptool-terrain-#{Process.pid}-#{Time.now.to_i}.mbtiles") tmp_mbtiles = File.join(Dir.tmpdir, "maptool-terrain-#{Process.pid}-#{Time.now.to_i}.mbtiles")
tmp_path = File.join(data_dir, "terrain.pmtiles.tmp") tmp_path = File.join(data_dir, "terrain.pmtiles.tmp")
write_terrarium_mbtiles(tmp_mbtiles, south, west, north, east, fetch_tile) write_terrarium_mbtiles(tmp_mbtiles, south, west, north, east, fetch_tile, progress)
run_pmtiles!("convert", tmp_mbtiles, tmp_path) run_pmtiles!("convert", tmp_mbtiles, tmp_path)
File.rename(tmp_path, File.join(data_dir, "terrain.pmtiles")) File.rename(tmp_path, File.join(data_dir, "terrain.pmtiles"))
ensure ensure
@@ -141,7 +145,7 @@ module Maptool
# park-sized region, slow for a big one. Add a small bounded-concurrency # park-sized region, slow for a big one. Add a small bounded-concurrency
# fetch pool if regenerating a large area turns out to matter in # fetch pool if regenerating a large area turns out to matter in
# practice. # practice.
def self.write_terrarium_mbtiles(path, south, west, north, east, fetch_tile) def self.write_terrarium_mbtiles(path, south, west, north, east, fetch_tile, progress)
db = SQLite3::Database.new(path) db = SQLite3::Database.new(path)
db.execute_batch(<<~SQL) db.execute_batch(<<~SQL)
CREATE TABLE metadata (name TEXT, value TEXT); CREATE TABLE metadata (name TEXT, value TEXT);
@@ -152,9 +156,19 @@ module Maptool
db.execute("INSERT INTO metadata (name, value) VALUES (?, ?)", [name, value]) db.execute("INSERT INTO metadata (name, value) VALUES (?, ?)", [name, value])
end end
(0..TERRAIN_MAX_ZOOM).each do |z| # tile_range per zoom computed once upfront (cheap — no fetching, just
tile_range(z, south, west, north, east).each do |x, y| # tile math) and reused below, so progress can report a real total
# from the very first call instead of only once the run finishes.
by_zoom = (0..TERRAIN_MAX_ZOOM).map { |z| [z, tile_range(z, south, west, north, east)] }
total = by_zoom.sum { |_z, tiles| tiles.size }
done = 0
progress.call(done, total)
by_zoom.each do |z, tiles|
tiles.each do |x, y|
bytes = fetch_tile.call(z, x, y) bytes = fetch_tile.call(z, x, y)
done += 1
progress.call(done, total)
next unless bytes next unless bytes
# MBTiles uses TMS row numbering (0 at the south) — the opposite # MBTiles uses TMS row numbering (0 at the south) — the opposite
+19
View File
@@ -9,6 +9,24 @@ function formatBytes(bytes) {
return mb >= 1024 ? `${(mb / 1024).toFixed(1)} GB` : `${mb.toFixed(1)} MB`; return mb >= 1024 ? `${(mb / 1024).toFixed(1)} GB` : `${mb.toFixed(1)} MB`;
} }
function formatElapsed(ms) {
const totalSeconds = Math.max(0, Math.floor(ms / 1000));
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`;
}
// "Building…" alone gave no way to tell a slow build (terrain.pmtiles can
// take minutes — lib/maptool/pmtiles_builder.rb fetches tiles one at a
// time) from a stuck one. Elapsed time alone answers "is it alive" for
// both files; tile counts (only terrain.pmtiles reports these) give an
// actual sense of how much is left.
function formatRunning(status) {
const elapsed = status.started_at ? ` (${formatElapsed(Date.now() - new Date(status.started_at).getTime())})` : "";
const tiles = status.tiles_total ? ` · ${status.tiles_done}/${status.tiles_total} tiles` : "";
return `Building…${elapsed}${tiles}`;
}
// One row of the Offline data panel — status text plus a "Regenerate for // One row of the Offline data panel — status text plus a "Regenerate for
// current view" button, for either offline.pmtiles or terrain.pmtiles. // current view" button, for either offline.pmtiles or terrain.pmtiles.
// Both files behave identically from here (same status/build/build-status // Both files behave identically from here (same status/build/build-status
@@ -103,6 +121,7 @@ class DataFileRow {
} }
if (status.state === "running") { if (status.state === "running") {
this._statusEl.textContent = formatRunning(status);
this._poll(); this._poll();
return; return;
} }
+28 -1
View File
@@ -113,7 +113,34 @@ class PmtilesBuilderTest < Minitest::Test
thread.join thread.join
assert_path_exists File.join(@data_dir, "terrain.pmtiles") assert_path_exists File.join(@data_dir, "terrain.pmtiles")
assert_equal "done", Maptool::PmtilesBuilder.terrain_status[:state] status = Maptool::PmtilesBuilder.terrain_status
assert_equal "done", status[:state]
assert_operator status[:tiles_total], :>, 0
assert_equal status[:tiles_total], status[:tiles_done]
end
# fetch_tile blocks after the very first call so the "running" progress
# can be observed mid-flight, deterministically, rather than only ever
# seeing the final done/total once the (very fast, faked) build finishes.
def test_build_terrain_reports_progress_while_running
ENV["PMTILES_BIN"] = @pmtiles_ok
gate = Queue.new
first_call = true
fetch_tile = lambda do |_z, _x, _y|
gate.pop if first_call
first_call = false
"bytes"
end
thread = Maptool::PmtilesBuilder.build_terrain([48.140, 17.100, 48.141, 17.101], data_dir: @data_dir, fetch_tile: fetch_tile)
20.times { break if Maptool::PmtilesBuilder.terrain_status[:tiles_total]; sleep 0.01 }
status = Maptool::PmtilesBuilder.terrain_status
assert_equal "running", status[:state]
assert_operator status[:tiles_total], :>, 0
assert_equal 0, status[:tiles_done]
ensure
gate.push(nil)
thread&.join
end end
def test_build_terrain_still_succeeds_when_some_tiles_are_missing def test_build_terrain_still_succeeds_when_some_tiles_are_missing