Add a country-based region picker for offline data
CI / check (push) Successful in 17s
CI / deploy (push) Successful in 12s

Replaces "regenerate for the current viewport" with what Locus/OsmAnd-
style apps give you: tick which countries you want offline (a Natural
Earth-derived checklist, public domain, vendored), Apply rebuilds
offline.pmtiles/terrain.pmtiles to cover the whole selection at once.
Multiple regions can be offline simultaneously (e.g. Slovakia + Austria
for a trip that crosses the border) — the selection itself is the
source of truth (data/offline_regions.json), not a set of per-country
files, so "remove a country" and "refresh a country" are both just
"change the selection, then Apply."

Confirmed against go-pmtiles' own source (not just its docs) before
committing to this shape: `extract --region=<geojson>` treats a
FeatureCollection's features as their union already (no dissolve/union
math needed on our side — pmtiles/region.go's UnmarshalRegion just
concatenates every feature's rings), and `merge` hard-errors on any
overlapping tile id between inputs (merge.go's prepareInputs). That
second fact rules out "extract each country separately, merge them
all" — adjacent countries' region-extracts would share border tiles,
since extraction copies whole tiles rather than clipping vector
content to the polygon. The actual fix: extract per *source archive*
(the Protomaps build, each intersecting Mapterhorn chunk), each call
covering the whole current selection via one --region= file — the
terrain pipeline already extracted-then-merged per source this way, it
just needed a multi-country region instead of a single bbox.
This commit is contained in:
randy committed 2026-09-11 12:55:56 +02:00
1 parent f89dea5805
commit 53c9a82af2
10 files changed
+442 -170

No files matched your search

+34 -12
View File
@@ -349,41 +349,63 @@ class AppTest < Minitest::Test
assert_equal 502, response.status
end
# --- offline regions ---
def test_offline_regions_lists_available_and_selected
body = json_body(request.get("/api/offline-regions"))
assert_operator body["available"].size, :>, 0
assert_includes body["available"], { "code" => "SK", "name" => "Slovakia" }
assert_equal [], body["selected"]
end
def test_offline_regions_post_requires_login
assert_equal 401, request.post("/api/offline-regions").status
end
def test_offline_regions_post_saves_the_selection
response = request.post("/api/offline-regions", params: { "codes" => %w[SK AT] }, **logged_in_env)
assert_equal 200, response.status
assert_equal %w[SK AT], json_body(request.get("/api/offline-regions"))["selected"]
end
def test_offline_regions_post_rejects_an_unknown_code
response = request.post("/api/offline-regions", params: { "codes" => ["XX"] }, **logged_in_env)
assert_equal 400, response.status
end
# --- pmtiles regeneration ---
#
# The full build (network + go-pmtiles) is covered in isolation, with
# fakes injected, by test_pmtiles_builder.rb — same "keep this offline"
# reasoning as the rest of this file. What's route-specific and worth
# covering here is the bbox validation and the 409-while-running guard,
# neither of which needs a real build to actually run.
# covering here is the empty-selection validation and the
# 409-while-running guard, neither of which needs a real build to
# actually run.
def test_offline_tiles_build_status_defaults_to_idle
assert_equal({ "state" => "idle" }, json_body(request.get("/api/offline-tiles/build-status")))
assert_equal({ "state" => "idle" }, json_body(request.get("/api/terrain-tiles/build-status")))
end
def test_offline_tiles_build_requires_a_bbox
def test_offline_tiles_build_is_409_with_no_selection
response = request.post("/api/offline-tiles/build", **logged_in_env)
assert_equal 400, response.status
assert_equal 409, response.status
end
def test_terrain_tiles_build_requires_a_bbox
def test_terrain_tiles_build_is_409_with_no_selection
response = request.post("/api/terrain-tiles/build", **logged_in_env)
assert_equal 400, response.status
assert_equal 409, response.status
end
# fetch_build_url blocks on a Queue, called directly (not through the
# route, which has no way to inject a fake) so the build is
# deterministically still "running" when the route call below happens.
def test_offline_tiles_build_is_409_when_one_is_already_running
Maptool::PmtilesBuilder.set_selected_regions(Maptool::App::DATA_DIR, ["SK"])
gate = Queue.new
thread = Maptool::PmtilesBuilder.build_offline([1, 2, 3, 4], data_dir: Maptool::App::DATA_DIR, fetch_build_url: -> { gate.pop })
thread = Maptool::PmtilesBuilder.build_offline(data_dir: Maptool::App::DATA_DIR, fetch_build_url: -> { gate.pop })
response = request.post(
"/api/offline-tiles/build",
params: { south: "1", west: "2", north: "3", east: "4" },
**logged_in_env,
)
response = request.post("/api/offline-tiles/build", **logged_in_env)
assert_equal 409, response.status
ensure
gate.push("unused")
+1
View File
@@ -66,6 +66,7 @@ module AppTestHelper
def teardown
FileUtils.rm_rf(Dir.glob(File.join(Maptool::App::DATA_DIR, "gpx", "*")))
File.write(File.join(Maptool::App::DATA_DIR, "livetrack.json"), "[]")
FileUtils.rm_f(File.join(Maptool::App::DATA_DIR, "offline_regions.json"))
Maptool::PmtilesBuilder.reset!
super
end
+80 -36
View File
@@ -11,13 +11,19 @@ require "fileutils"
# CLI (PMTILES_BIN) stands in for the real go-pmtiles binary, logging each
# invocation's subcommand + first arg to PMTILES_LOG so tests can assert on
# what actually got called (e.g. "was merge invoked, and with what").
#
# Country codes below are real ones from the vendored
# lib/maptool/data/europe_countries.geojson (Slovakia, Iceland, Spain) —
# chosen so their real bboxes fall cleanly inside/outside the fake
# manifest's chunk extents below, without needing a second fake geometry
# source.
class PmtilesBuilderTest < Minitest::Test
def setup
@data_dir = Dir.mktmpdir("maptool-pmtiles-test-data")
@bin_dir = Dir.mktmpdir("maptool-pmtiles-test-bin")
@log = File.join(@bin_dir, "pmtiles.log")
# Output path: $3 for `extract SOURCE OUTPUT --bbox=... --maxzoom=...`
# Output path: $3 for `extract SOURCE OUTPUT --region=... --maxzoom=...`
# and `convert INPUT OUTPUT` alike; the *last* arg for `merge INPUT_1
# [INPUT_2 ...] OUTPUT`, since its input count varies.
@pmtiles_ok = File.join(@bin_dir, "pmtiles-ok")
@@ -57,15 +63,17 @@ class PmtilesBuilderTest < Minitest::Test
File.exist?(@log) ? File.readlines(@log, chomp: true) : []
end
# planet.pmtiles always covers the whole world (z0-12); chunk-a/chunk-b
# are disjoint small regions (z13+), matching Mapterhorn's real shape
# (see download_urls.json) closely enough to exercise bbox_intersects?
# and the extract-then-maybe-merge logic without a real manifest fetch.
def select_regions(*codes)
Maptool::PmtilesBuilder.set_selected_regions(@data_dir, codes)
end
# chunk-a covers Slovakia's real extent, chunk-b covers Iceland's —
# Spain falls in neither, for the "no chunk intersects" case.
def fake_manifest
[
{ "name" => "planet.pmtiles", "url" => "https://example.com/planet.pmtiles", "min_lon" => -180.0, "min_lat" => -85.0, "max_lon" => 180.0, "max_lat" => 85.0, "min_zoom" => 0, "max_zoom" => 12 },
{ "name" => "chunk-a.pmtiles", "url" => "https://example.com/chunk-a.pmtiles", "min_lon" => 17.0, "min_lat" => 48.0, "max_lon" => 17.5, "max_lat" => 48.5, "min_zoom" => 13, "max_zoom" => 15 },
{ "name" => "chunk-b.pmtiles", "url" => "https://example.com/chunk-b.pmtiles", "min_lon" => 20.0, "min_lat" => 50.0, "max_lon" => 20.5, "max_lat" => 50.5, "min_zoom" => 13, "max_zoom" => 15 },
{ "name" => "chunk-a.pmtiles", "url" => "https://example.com/chunk-a.pmtiles", "min_lon" => 16.0, "min_lat" => 47.0, "max_lon" => 23.0, "max_lat" => 50.0, "min_zoom" => 13, "max_zoom" => 15 },
{ "name" => "chunk-b.pmtiles", "url" => "https://example.com/chunk-b.pmtiles", "min_lon" => -25.0, "min_lat" => 63.0, "max_lon" => -13.0, "max_lat" => 67.0, "min_zoom" => 13, "max_zoom" => 15 },
]
end
@@ -74,27 +82,51 @@ class PmtilesBuilderTest < Minitest::Test
assert_equal({ state: "idle" }, Maptool::PmtilesBuilder.terrain_status)
end
# --- region selection ---
def test_available_regions_includes_known_countries
codes = Maptool::PmtilesBuilder.available_regions.map { |r| r[:code] }
assert_includes codes, "SK"
assert_includes codes, "IS"
end
def test_selected_regions_defaults_to_empty
assert_empty Maptool::PmtilesBuilder.selected_regions(@data_dir)
end
def test_set_selected_regions_round_trips
select_regions("SK", "AT")
assert_equal ["SK", "AT"], Maptool::PmtilesBuilder.selected_regions(@data_dir)
end
def test_set_selected_regions_rejects_an_unknown_code
assert_raises(Maptool::PmtilesBuilder::BuildError) { select_regions("SK", "XX") }
end
# --- offline.pmtiles ---
def test_build_offline_raises_with_no_selection
assert_raises(Maptool::PmtilesBuilder::BuildError) do
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, fetch_build_url: -> { "unused" })
end
end
def test_build_offline_writes_the_file_and_reports_done
select_regions("SK")
ENV["PMTILES_BIN"] = @pmtiles_ok
thread = Maptool::PmtilesBuilder.build_offline(
[1, 2, 3, 4],
data_dir: @data_dir,
fetch_build_url: -> { "https://example.com/fake.pmtiles" },
)
thread = Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, fetch_build_url: -> { "https://example.com/fake.pmtiles" })
thread.join
assert_path_exists File.join(@data_dir, "offline.pmtiles")
refute_path_exists File.join(@data_dir, "offline.pmtiles.tmp")
assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state]
assert_equal ["extract https://example.com/fake.pmtiles"], log_lines
end
def test_build_offline_reports_an_error_status_on_non_zero_exit
select_regions("SK")
ENV["PMTILES_BIN"] = @pmtiles_fail
thread = Maptool::PmtilesBuilder.build_offline(
[1, 2, 3, 4],
data_dir: @data_dir,
fetch_build_url: -> { "https://example.com/fake.pmtiles" },
)
thread = Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, fetch_build_url: -> { "https://example.com/fake.pmtiles" })
thread.join
refute_path_exists File.join(@data_dir, "offline.pmtiles")
@@ -104,12 +136,9 @@ class PmtilesBuilderTest < Minitest::Test
end
def test_build_offline_reports_a_clear_error_when_the_binary_is_missing
select_regions("SK")
ENV["PMTILES_BIN"] = File.join(@bin_dir, "does-not-exist")
thread = Maptool::PmtilesBuilder.build_offline(
[1, 2, 3, 4],
data_dir: @data_dir,
fetch_build_url: -> { "https://example.com/fake.pmtiles" },
)
thread = Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, fetch_build_url: -> { "https://example.com/fake.pmtiles" })
thread.join
status = Maptool::PmtilesBuilder.offline_status
@@ -121,23 +150,33 @@ class PmtilesBuilderTest < Minitest::Test
# still "running" when the second call happens, rather than racing a real
# one to see which state wins.
def test_build_offline_raises_when_one_is_already_running
select_regions("SK")
ENV["PMTILES_BIN"] = @pmtiles_ok
gate = Queue.new
thread = Maptool::PmtilesBuilder.build_offline([1, 2, 3, 4], data_dir: @data_dir, fetch_build_url: -> { gate.pop })
thread = Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, fetch_build_url: -> { gate.pop })
assert_raises(Maptool::PmtilesBuilder::BuildError) do
Maptool::PmtilesBuilder.build_offline([1, 2, 3, 4], data_dir: @data_dir, fetch_build_url: -> { "unused" })
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, fetch_build_url: -> { "unused" })
end
ensure
gate.push("https://example.com/fake.pmtiles")
thread&.join
end
# A bbox nowhere near chunk-a/chunk-b — only planet.pmtiles matches, so
# --- terrain.pmtiles ---
def test_build_terrain_raises_with_no_selection
assert_raises(Maptool::PmtilesBuilder::BuildError) do
Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
end
end
# Spain intersects neither fake chunk — only planet.pmtiles matches, so
# this should be a single extract straight to the final path, no merge.
def test_build_terrain_extracts_only_planet_when_no_chunk_intersects
select_regions("ES")
ENV["PMTILES_BIN"] = @pmtiles_ok
thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
thread = Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
thread.join
assert_path_exists File.join(@data_dir, "terrain.pmtiles")
@@ -145,11 +184,12 @@ class PmtilesBuilderTest < Minitest::Test
assert_equal ["extract https://example.com/planet.pmtiles"], log_lines
end
# A bbox inside chunk-a's extent — planet.pmtiles (always) plus that one
# chunk get extracted, then merged.
# Slovakia intersects chunk-a only — planet.pmtiles (always) plus that
# one chunk get extracted, then merged.
def test_build_terrain_extracts_and_merges_a_single_intersecting_chunk
select_regions("SK")
ENV["PMTILES_BIN"] = @pmtiles_ok
thread = Maptool::PmtilesBuilder.build_terrain([48.1, 17.1, 48.2, 17.2], data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
thread = Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
thread.join
assert_path_exists File.join(@data_dir, "terrain.pmtiles")
@@ -160,11 +200,12 @@ class PmtilesBuilderTest < Minitest::Test
assert_equal 1, merged.size
end
# A bbox spanning both chunk-a and chunk-b — planet + both chunks, one
# 3-way merge.
# Slovakia + Iceland together intersect both fake chunks — planet + both
# chunks, one 3-way merge.
def test_build_terrain_merges_multiple_intersecting_chunks
select_regions("SK", "IS")
ENV["PMTILES_BIN"] = @pmtiles_ok
thread = Maptool::PmtilesBuilder.build_terrain([47.9, 16.9, 50.6, 20.6], data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
thread = Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
thread.join
assert_equal "done", Maptool::PmtilesBuilder.terrain_status[:state]
@@ -174,8 +215,9 @@ class PmtilesBuilderTest < Minitest::Test
end
def test_build_terrain_reports_a_clear_error_when_the_manifest_is_missing_planet
select_regions("ES")
ENV["PMTILES_BIN"] = @pmtiles_ok
thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { fake_manifest.reject { |i| i["name"] == "planet.pmtiles" } })
thread = Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { fake_manifest.reject { |i| i["name"] == "planet.pmtiles" } })
thread.join
status = Maptool::PmtilesBuilder.terrain_status
@@ -184,8 +226,9 @@ class PmtilesBuilderTest < Minitest::Test
end
def test_build_terrain_reports_an_error_status_on_non_zero_exit
select_regions("ES")
ENV["PMTILES_BIN"] = @pmtiles_fail
thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
thread = Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { fake_manifest })
thread.join
refute_path_exists File.join(@data_dir, "terrain.pmtiles")
@@ -195,12 +238,13 @@ class PmtilesBuilderTest < Minitest::Test
end
def test_build_terrain_raises_when_one_is_already_running
select_regions("ES")
ENV["PMTILES_BIN"] = @pmtiles_ok
gate = Queue.new
thread = Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { gate.pop })
thread = Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { gate.pop })
assert_raises(Maptool::PmtilesBuilder::BuildError) do
Maptool::PmtilesBuilder.build_terrain([0, 0, 1, 1], data_dir: @data_dir, fetch_manifest: -> { "unused" })
Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { "unused" })
end
ensure
gate.push(fake_manifest)