Cache per-country Geofabrik extracts so Apply doesn't redownload everything

Every region-selection change re-downloaded every selected country's
.osm.pbf from scratch, even ones already fetched by a prior build —
painful for multi-GB countries. Extracts now persist in
data/geofabrik_cache/, keyed by country, and are only re-fetched when
Geofabrik's Last-Modified for that country actually moved on (checked
with a cheap HEAD) or the file isn't cached yet.
This commit is contained in:
randy committed 2026-09-11 22:52:16 +02:00
1 parent 077b44ece1
commit 007dee7341
4 files changed
+131 -17

No files matched your search

+73 -2
View File
@@ -102,8 +102,12 @@ class PmtilesBuilderTest < Minitest::Test
# download_extract stub: writes a marker file (same "fake" content the
# pmtiles/osmium/java fixtures use) instead of hitting Geofabrik, and
# returns a fixed Last-Modified so sidecar-meta assertions are stable.
def fake_download(last_modified: "Wed, 10 Sep 2026 08:00:01 GMT")
->(_url, dest) {
# `calls` (an Array, shared across builds if the caller wants it) records
# every URL actually downloaded, for asserting the per-country cache
# skipped (or didn't skip) a re-download.
def fake_download(last_modified: "Wed, 10 Sep 2026 08:00:01 GMT", calls: nil)
->(url, dest) {
calls << url if calls
File.write(dest, "fake")
last_modified
}
@@ -189,6 +193,73 @@ class PmtilesBuilderTest < Minitest::Test
assert_equal %w[SK AT], codes
end
# The actual bug report this covers: hitting "Apply" after tweaking the
# region selection used to re-download every selected country's .osm.pbf
# from scratch, even ones already fetched by an earlier build.
def test_build_offline_skips_redownloading_an_unchanged_cached_country
select_regions("SK")
calls = []
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
assert_equal 1, calls.size
Maptool::PmtilesBuilder.build_offline(
data_dir: @data_dir,
download_extract: fake_download(calls: calls),
fetch_last_modified: ->(_url) { "Wed, 10 Sep 2026 08:00:01 GMT" },
).join
assert_equal 1, calls.size
assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state]
end
def test_build_offline_redownloads_a_country_whose_extract_changed
select_regions("SK")
calls = []
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
Maptool::PmtilesBuilder.build_offline(
data_dir: @data_dir,
download_extract: fake_download(calls: calls, last_modified: "Thu, 11 Sep 2026 08:00:01 GMT"),
fetch_last_modified: ->(_url) { "Thu, 11 Sep 2026 08:00:01 GMT" },
).join
assert_equal 2, calls.size
end
# A cache-freshness check that itself fails (Geofabrik unreachable) should
# fall back to re-downloading rather than aborting the build.
def test_build_offline_redownloads_when_the_cache_check_fails
select_regions("SK")
calls = []
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
fetch_last_modified = ->(_url) { raise Maptool::PmtilesBuilder::BuildError, "could not reach geofabrik" }
Maptool::PmtilesBuilder.build_offline(
data_dir: @data_dir,
download_extract: fake_download(calls: calls),
fetch_last_modified: fetch_last_modified,
).join
assert_equal 2, calls.size
assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state]
end
def test_build_offline_only_downloads_a_newly_added_country
select_regions("SK")
calls = []
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
select_regions("SK", "AT")
Maptool::PmtilesBuilder.build_offline(
data_dir: @data_dir,
download_extract: fake_download(calls: calls),
fetch_last_modified: ->(_url) { "Wed, 10 Sep 2026 08:00:01 GMT" },
).join
assert_equal 1, calls.count { |url| url.include?("slovakia") }
assert_equal 1, calls.count { |url| url.include?("austria") }
end
def test_offline_meta_is_nil_before_any_build
assert_nil Maptool::PmtilesBuilder.offline_meta(@data_dir)
end