Cache per-country Geofabrik extracts so Apply doesn't redownload everything
Every region-selection change re-downloaded every selected country's .osm.pbf from scratch, even ones already fetched by a prior build — painful for multi-GB countries. Extracts now persist in data/geofabrik_cache/, keyed by country, and are only re-fetched when Geofabrik's Last-Modified for that country actually moved on (checked with a cheap HEAD) or the file isn't cached yet.
This commit is contained in:
1 parent
077b44ece1
commit
007dee7341
4 files changed
+131
-17
No files matched your search
@@ -102,8 +102,12 @@ class PmtilesBuilderTest < Minitest::Test
|
||||
# download_extract stub: writes a marker file (same "fake" content the
|
||||
# pmtiles/osmium/java fixtures use) instead of hitting Geofabrik, and
|
||||
# returns a fixed Last-Modified so sidecar-meta assertions are stable.
|
||||
def fake_download(last_modified: "Wed, 10 Sep 2026 08:00:01 GMT")
|
||||
->(_url, dest) {
|
||||
# `calls` (an Array, shared across builds if the caller wants it) records
|
||||
# every URL actually downloaded, for asserting the per-country cache
|
||||
# skipped (or didn't skip) a re-download.
|
||||
def fake_download(last_modified: "Wed, 10 Sep 2026 08:00:01 GMT", calls: nil)
|
||||
->(url, dest) {
|
||||
calls << url if calls
|
||||
File.write(dest, "fake")
|
||||
last_modified
|
||||
}
|
||||
@@ -189,6 +193,73 @@ class PmtilesBuilderTest < Minitest::Test
|
||||
assert_equal %w[SK AT], codes
|
||||
end
|
||||
|
||||
# The actual bug report this covers: hitting "Apply" after tweaking the
|
||||
# region selection used to re-download every selected country's .osm.pbf
|
||||
# from scratch, even ones already fetched by an earlier build.
|
||||
def test_build_offline_skips_redownloading_an_unchanged_cached_country
|
||||
select_regions("SK")
|
||||
calls = []
|
||||
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
|
||||
assert_equal 1, calls.size
|
||||
|
||||
Maptool::PmtilesBuilder.build_offline(
|
||||
data_dir: @data_dir,
|
||||
download_extract: fake_download(calls: calls),
|
||||
fetch_last_modified: ->(_url) { "Wed, 10 Sep 2026 08:00:01 GMT" },
|
||||
).join
|
||||
|
||||
assert_equal 1, calls.size
|
||||
assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state]
|
||||
end
|
||||
|
||||
def test_build_offline_redownloads_a_country_whose_extract_changed
|
||||
select_regions("SK")
|
||||
calls = []
|
||||
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
|
||||
|
||||
Maptool::PmtilesBuilder.build_offline(
|
||||
data_dir: @data_dir,
|
||||
download_extract: fake_download(calls: calls, last_modified: "Thu, 11 Sep 2026 08:00:01 GMT"),
|
||||
fetch_last_modified: ->(_url) { "Thu, 11 Sep 2026 08:00:01 GMT" },
|
||||
).join
|
||||
|
||||
assert_equal 2, calls.size
|
||||
end
|
||||
|
||||
# A cache-freshness check that itself fails (Geofabrik unreachable) should
|
||||
# fall back to re-downloading rather than aborting the build.
|
||||
def test_build_offline_redownloads_when_the_cache_check_fails
|
||||
select_regions("SK")
|
||||
calls = []
|
||||
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
|
||||
|
||||
fetch_last_modified = ->(_url) { raise Maptool::PmtilesBuilder::BuildError, "could not reach geofabrik" }
|
||||
Maptool::PmtilesBuilder.build_offline(
|
||||
data_dir: @data_dir,
|
||||
download_extract: fake_download(calls: calls),
|
||||
fetch_last_modified: fetch_last_modified,
|
||||
).join
|
||||
|
||||
assert_equal 2, calls.size
|
||||
assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state]
|
||||
end
|
||||
|
||||
def test_build_offline_only_downloads_a_newly_added_country
|
||||
select_regions("SK")
|
||||
calls = []
|
||||
Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download(calls: calls)).join
|
||||
|
||||
select_regions("SK", "AT")
|
||||
Maptool::PmtilesBuilder.build_offline(
|
||||
data_dir: @data_dir,
|
||||
download_extract: fake_download(calls: calls),
|
||||
fetch_last_modified: ->(_url) { "Wed, 10 Sep 2026 08:00:01 GMT" },
|
||||
).join
|
||||
|
||||
assert_equal 1, calls.count { |url| url.include?("slovakia") }
|
||||
assert_equal 1, calls.count { |url| url.include?("austria") }
|
||||
end
|
||||
|
||||
def test_offline_meta_is_nil_before_any_build
|
||||
assert_nil Maptool::PmtilesBuilder.offline_meta(@data_dir)
|
||||
end
|
||||
|
||||
Reference in new issue
Block a user