Build terrain.pmtiles from Mapterhorn instead of fetching tiles ourselves
Mapterhorn (mapterhorn.com, Protomaps-affiliated) publishes pre-built Terrarium-encoded terrain as PMTiles: a global planet.pmtiles (z0-12) plus ~450 regional chunks (z13+). Same pmtiles extract --bbox=... mechanism offline.pmtiles already uses, just against their files — confirmed live against their real manifest and the documented Interlaken example. Replaces fetching every individual tile ourselves (even after this session's earlier concurrency fix, still tens of minutes for a real region) with 1-3 extract calls plus a merge when the bbox spans more than one chunk: seconds, not minutes. Removes CONCURRENT_TILE_FETCHES, write_terrarium_mbtiles, tile_range, lng/lat_to_tile_x/y, fetch_terrarium_tile, and the sqlite3 gem entirely — none of it is needed once the tiles arrive pre-packaged. dem.js's point-elevation lookup (shadow.js's terrain-shadow overlay) hardcoded a 256px PNG tile assumption that Mapterhorn's 512px WebP tiles would have silently broken — now reads the actually-decoded tile's own size and mime type instead of assuming either.
This commit is contained in:
randy committed
1 parent
d041c21270
commit
f89dea5805
8 files changed
+224
-234
No files matched your search
+101
-56
@@ -3,22 +3,37 @@ require "maptool/pmtiles_builder"
|
||||
require "tmpdir"
|
||||
require "fileutils"
|
||||
|
||||
# Network calls (the protomaps build index, individual Terrarium tile
|
||||
# fetches) are never exercised here — same "keep this offline/deterministic"
|
||||
# reasoning test_helper.rb already gives for /api/plugins staying off the
|
||||
# real Overpass API — build_offline/build_terrain take fetch_build_url/
|
||||
# fetch_tile precisely so tests can swap those out. A fake `pmtiles` CLI
|
||||
# (PMTILES_BIN) stands in for the real go-pmtiles binary.
|
||||
# Network calls (the protomaps build index, the mapterhorn manifest) are
|
||||
# never exercised here — same "keep this offline/deterministic" reasoning
|
||||
# test_helper.rb already gives for /api/plugins staying off the real
|
||||
# Overpass API — build_offline/build_terrain take fetch_build_url/
|
||||
# fetch_manifest precisely so tests can swap those out. A fake `pmtiles`
|
||||
# CLI (PMTILES_BIN) stands in for the real go-pmtiles binary, logging each
|
||||
# invocation's subcommand + first arg to PMTILES_LOG so tests can assert on
|
||||
# what actually got called (e.g. "was merge invoked, and with what").
|
||||
class PmtilesBuilderTest < Minitest::Test
|
||||
def setup
|
||||
@data_dir = Dir.mktmpdir("maptool-pmtiles-test-data")
|
||||
@bin_dir = Dir.mktmpdir("maptool-pmtiles-test-bin")
|
||||
@log = File.join(@bin_dir, "pmtiles.log")
|
||||
|
||||
# $3 is the output path either way: `extract SOURCE OUTPUT --bbox=...
|
||||
# --maxzoom=...` and `convert INPUT OUTPUT` both put it 3rd, even though
|
||||
# it's not the *last* arg in the extract case (the --flags follow it).
|
||||
# Output path: $3 for `extract SOURCE OUTPUT --bbox=... --maxzoom=...`
|
||||
# and `convert INPUT OUTPUT` alike; the *last* arg for `merge INPUT_1
|
||||
# [INPUT_2 ...] OUTPUT`, since its input count varies.
|
||||
@pmtiles_ok = File.join(@bin_dir, "pmtiles-ok")
|
||||
File.write(@pmtiles_ok, "#!/bin/sh\necho fake > \"$3\"\n")
|
||||
File.write(@pmtiles_ok, <<~SH)
|
||||
#!/bin/sh
|
||||
echo "$1 $2" >> "$PMTILES_LOG"
|
||||
case "$1" in
|
||||
merge)
|
||||
for last; do :; done
|
||||
echo fake > "$last"
|
||||
;;
|
||||
*)
|
||||
echo fake > "$3"
|
||||
;;
|
||||
esac
|
||||
SH
|
||||
File.chmod(0o755, @pmtiles_ok)
|
||||
|
||||
@pmtiles_fail = File.join(@bin_dir, "pmtiles-fail")
|
||||
@@ -26,6 +41,7 @@ class PmtilesBuilderTest < Minitest::Test
|
||||
File.chmod(0o755, @pmtiles_fail)
|
||||
|
||||
@original_pmtiles_bin = ENV["PMTILES_BIN"]
|
||||
ENV["PMTILES_LOG"] = @log
|
||||
Maptool::PmtilesBuilder.reset!
|
||||
end
|
||||
|
||||
@@ -33,9 +49,26 @@ class PmtilesBuilderTest < Minitest::Test
|
||||
FileUtils.remove_entry(@data_dir)
|
||||
FileUtils.remove_entry(@bin_dir)
|
||||
ENV["PMTILES_BIN"] = @original_pmtiles_bin
|
||||
ENV.delete("PMTILES_LOG")
|
||||
Maptool::PmtilesBuilder.reset!
|
||||
end
|
||||
|
||||
def log_lines
|
||||
File.exist?(@log) ? File.readlines(@log, chomp: true) : []
|
||||
end
|
||||
|
||||
# planet.pmtiles always covers the whole world (z0-12); chunk-a/chunk-b
|
||||
# are disjoint small regions (z13+), matching Mapterhorn's real shape
|
||||
# (see download_urls.json) closely enough to exercise bbox_intersects?
|
||||
# and the extract-then-maybe-merge logic without a real manifest fetch.
|
||||
def fake_manifest
|
||||
[
|
||||
{ "name" => "planet.pmtiles", "url" => "https://example.com/planet.pmtiles", "min_lon" => -180.0, "min_lat" => -85.0, "max_lon" => 180.0, "max_lat" => 85.0, "min_zoom" => 0, "max_zoom" => 12 },
|
||||
{ "name" => "chunk-a.pmtiles", "url" => "https://example.com/chunk-a.pmtiles", "min_lon" => 17.0, "min_lat" => 48.0, "max_lon" => 17.5, "max_lat" => 48.5, "min_zoom" => 13, "max_zoom" => 15 },
|
||||
{ "name" => "chunk-b.pmtiles", "url" => "https://example.com/chunk-b.pmtiles", "min_lon" => 20.0, "min_lat" => 50.0, "max_lon" => 20.5, "max_lat" => 50.5, "min_zoom" => 13, "max_zoom" => 15 },
|
||||
]
|
||||
end
|
||||
|
||||
def test_offline_and_terrain_status_start_idle
|
||||
assert_equal({ state: "idle" }, Maptool::PmtilesBuilder.offline_status)
|
||||
assert_equal({ state: "idle" }, Maptool::PmtilesBuilder.terrain_status)
|
||||
@@ -100,65 +133,77 @@ class PmtilesBuilderTest < Minitest::Test
|
||||
thread&.join
|
||||
end
|
||||
|
||||
# A tiny real-world bbox (a few blocks of central Bratislava) keeps the
|
||||
# z0..15 tile_range small at every zoom level, so this stays fast even
|
||||
# though it iterates all 16 levels.
|
||||
def test_build_terrain_packs_fetched_tiles_and_converts
|
||||
# A bbox nowhere near chunk-a/chunk-b — only planet.pmtiles matches, so
|
||||
# this should be a single extract straight to the final path, no merge.
|
||||
def test_build_terrain_extracts_only_planet_when_no_chunk_intersects
|
||||
ENV["PMTILES_BIN"] = @pmtiles_ok
|
||||
thread = Maptool::PmtilesBuilder.build_terrain(
|
||||
[48.140, 17.100, 48.141, 17.101],
|
||||
data_dir: @data_dir,
|
||||
fetch_tile: ->(_z, _x, _y) { "not a real png, write_terrarium_mbtiles just stores opaque bytes" },
|
||||
)
|
||||
thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
|
||||
thread.join
|
||||
|
||||
assert_path_exists File.join(@data_dir, "terrain.pmtiles")
|
||||
status = Maptool::PmtilesBuilder.terrain_status
|
||||
assert_equal "done", status[:state]
|
||||
assert_operator status[:tiles_total], :>, 0
|
||||
assert_equal status[:tiles_total], status[:tiles_done]
|
||||
assert_equal "done", Maptool::PmtilesBuilder.terrain_status[:state]
|
||||
assert_equal ["extract https://example.com/planet.pmtiles"], log_lines
|
||||
end
|
||||
|
||||
# Every fetch_tile call blocks on a gate until explicitly told not to —
|
||||
# with CONCURRENT_TILE_FETCHES workers pulling from a shared queue, that
|
||||
# holds the whole pool at "in flight, none finished yet" deterministically
|
||||
# (a big enough bbox that z0..15 yields far more tiles than worker slots,
|
||||
# so most of the job is still queued up behind the blocked pool).
|
||||
def test_build_terrain_reports_progress_while_running
|
||||
# A bbox inside chunk-a's extent — planet.pmtiles (always) plus that one
|
||||
# chunk get extracted, then merged.
|
||||
def test_build_terrain_extracts_and_merges_a_single_intersecting_chunk
|
||||
ENV["PMTILES_BIN"] = @pmtiles_ok
|
||||
concurrency = Maptool::PmtilesBuilder::CONCURRENT_TILE_FETCHES
|
||||
gate = Queue.new
|
||||
unblocked = false
|
||||
fetch_tile = lambda do |_z, _x, _y|
|
||||
gate.pop unless unblocked
|
||||
"bytes"
|
||||
end
|
||||
thread = Maptool::PmtilesBuilder.build_terrain([47.5, 16.5, 48.5, 17.5], data_dir: @data_dir, fetch_tile: fetch_tile)
|
||||
thread = Maptool::PmtilesBuilder.build_terrain([48.1, 17.1, 48.2, 17.2], data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
|
||||
thread.join
|
||||
|
||||
# Let exactly one batch through (fills every worker's first call) then
|
||||
# check before releasing the rest — done should be capped at the pool
|
||||
# size, nowhere near the real total.
|
||||
concurrency.times { gate.push(nil) }
|
||||
sleep 0.05
|
||||
status = Maptool::PmtilesBuilder.terrain_status
|
||||
assert_equal "running", status[:state]
|
||||
assert_operator status[:tiles_total], :>, concurrency
|
||||
assert_operator status[:tiles_done], :<, status[:tiles_total]
|
||||
ensure
|
||||
unblocked = true
|
||||
concurrency.times { gate.push(nil) } # release whatever's still blocked in an in-progress gate.pop
|
||||
thread&.join
|
||||
assert_path_exists File.join(@data_dir, "terrain.pmtiles")
|
||||
assert_equal "done", Maptool::PmtilesBuilder.terrain_status[:state]
|
||||
extracted = log_lines.select { |line| line.start_with?("extract ") }
|
||||
merged = log_lines.select { |line| line.start_with?("merge ") }
|
||||
assert_equal ["extract https://example.com/planet.pmtiles", "extract https://example.com/chunk-a.pmtiles"], extracted
|
||||
assert_equal 1, merged.size
|
||||
end
|
||||
|
||||
def test_build_terrain_still_succeeds_when_some_tiles_are_missing
|
||||
# A bbox spanning both chunk-a and chunk-b — planet + both chunks, one
|
||||
# 3-way merge.
|
||||
def test_build_terrain_merges_multiple_intersecting_chunks
|
||||
ENV["PMTILES_BIN"] = @pmtiles_ok
|
||||
thread = Maptool::PmtilesBuilder.build_terrain(
|
||||
[48.140, 17.100, 48.141, 17.101],
|
||||
data_dir: @data_dir,
|
||||
fetch_tile: ->(_z, _x, _y) { nil }, # every tile "missing" — should skip, not fail, the same way HillshadeControl's own source tolerates gaps
|
||||
)
|
||||
thread = Maptool::PmtilesBuilder.build_terrain([47.9, 16.9, 50.6, 20.6], data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
|
||||
thread.join
|
||||
|
||||
assert_equal "done", Maptool::PmtilesBuilder.terrain_status[:state]
|
||||
extracted = log_lines.count { |line| line.start_with?("extract ") }
|
||||
assert_equal 3, extracted # planet + chunk-a + chunk-b
|
||||
assert_equal 1, log_lines.count { |line| line.start_with?("merge ") }
|
||||
end
|
||||
|
||||
def test_build_terrain_reports_a_clear_error_when_the_manifest_is_missing_planet
|
||||
ENV["PMTILES_BIN"] = @pmtiles_ok
|
||||
thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { fake_manifest.reject { |i| i["name"] == "planet.pmtiles" } })
|
||||
thread.join
|
||||
|
||||
status = Maptool::PmtilesBuilder.terrain_status
|
||||
assert_equal "error", status[:state]
|
||||
assert_match(/missing planet\.pmtiles/, status[:message])
|
||||
end
|
||||
|
||||
def test_build_terrain_reports_an_error_status_on_non_zero_exit
|
||||
ENV["PMTILES_BIN"] = @pmtiles_fail
|
||||
thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
|
||||
thread.join
|
||||
|
||||
refute_path_exists File.join(@data_dir, "terrain.pmtiles")
|
||||
status = Maptool::PmtilesBuilder.terrain_status
|
||||
assert_equal "error", status[:state]
|
||||
assert_match(/exited with status/, status[:message])
|
||||
end
|
||||
|
||||
def test_build_terrain_raises_when_one_is_already_running
|
||||
ENV["PMTILES_BIN"] = @pmtiles_ok
|
||||
gate = Queue.new
|
||||
thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { gate.pop })
|
||||
|
||||
assert_raises(Maptool::PmtilesBuilder::BuildError) do
|
||||
Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { "unused" })
|
||||
end
|
||||
ensure
|
||||
gate.push(fake_manifest)
|
||||
thread&.join
|
||||
end
|
||||
end
|
||||
Reference in new issue
Block a user