Cache per-country Geofabrik extracts so Apply doesn't redownload everything

Every region-selection change re-downloaded every selected country's
.osm.pbf from scratch, even ones already fetched by a prior build —
painful for multi-GB countries. Extracts now persist in
data/geofabrik_cache/, keyed by country, and are only re-fetched when
Geofabrik's Last-Modified for that country actually moved on (checked
with a cheap HEAD) or the file isn't cached yet.
This commit is contained in:
randy committed 2026-09-11 22:52:16 +02:00
1 parent 077b44ece1
commit 007dee7341
4 files changed
+131 -17

No files matched your search

+44 -11
View File
@@ -179,17 +179,17 @@ module Maptool
# finishes — raises BuildError (the caller turns this into an error
# response) if nothing's selected yet, or one's already running.
#
# download_extract exists as an injectable seam so tests can exercise
# the rest of this (scratch dir, osmium/planetiler invocation, atomic
# rename) without a real network call — same reasoning test_helper.rb
# already gives for keeping /api/plugins route tests off the real
# Overpass API.
def self.build_offline(data_dir:, download_extract: method(:download_extract))
# download_extract/fetch_last_modified exist as injectable seams so
# tests can exercise the rest of this (scratch dir, osmium/planetiler
# invocation, atomic rename, the per-country cache below) without a
# real network call — same reasoning test_helper.rb already gives for
# keeping /api/plugins route tests off the real Overpass API.
def self.build_offline(data_dir:, download_extract: method(:download_extract), fetch_last_modified: method(:fetch_geofabrik_last_modified))
codes = selected_regions(data_dir)
raise BuildError, "select at least one region first" if codes.empty?
begin_run!(OFFLINE_MUTEX, OFFLINE_STATUS)
Thread.new { run_job(OFFLINE_MUTEX, OFFLINE_STATUS) { extract_offline(codes, data_dir, download_extract) } }
Thread.new { run_job(OFFLINE_MUTEX, OFFLINE_STATUS) { extract_offline(codes, data_dir, download_extract, fetch_last_modified) } }
end
# fetch_manifest: same injectable-seam reasoning as download_extract
@@ -228,13 +228,11 @@ module Maptool
# tiles.openfreemap.org's Liberty style already expects, so
# public/js/map.js's offlineStyle() can reuse ONLINE_STYLE_PATCH
# unchanged instead of a separate protomaps-themes-base palette.
def self.extract_offline(codes, data_dir, download_extract)
def self.extract_offline(codes, data_dir, download_extract, fetch_last_modified)
scratch_dir = Dir.mktmpdir("maptool-offline")
sources = codes.map do |code|
slug = GEOFABRIK_SLUGS.fetch(code) { raise BuildError, "no Geofabrik source known for region #{code}" }
dest = File.join(scratch_dir, "#{code}.osm.pbf")
last_modified = download_extract.call(geofabrik_url(slug), dest)
{ "code" => code, "slug" => slug, "last_modified" => last_modified, "path" => dest }
cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
end
osm_path =
@@ -261,6 +259,41 @@ module Maptool
end
private_class_method :geofabrik_url
# Per-country .osm.pbf cache, kept across builds under data_dir (not
# scratch_dir, which is wiped every run) — changing the region
# selection re-downloads only the countries that are new to it, and
# re-running the same selection re-downloads nothing unless Geofabrik's
# own Last-Modified for that country moved on. A stamp file next to
# each cached extract records the Last-Modified it was fetched with;
# a HEAD check (cheap) decides whether the multi-GB GET (not cheap) is
# skippable.
def self.cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
cache_dir = geofabrik_cache_dir(data_dir)
FileUtils.mkdir_p(cache_dir)
path = File.join(cache_dir, "#{code}.osm.pbf")
stamp_path = "#{path}.last_modified"
url = geofabrik_url(slug)
if File.exist?(path) && File.exist?(stamp_path)
current = begin
fetch_last_modified.call(url)
rescue BuildError
nil
end
return { "code" => code, "slug" => slug, "last_modified" => current, "path" => path } if current && current == File.read(stamp_path)
end
last_modified = download_extract.call(url, path)
File.write(stamp_path, last_modified.to_s)
{ "code" => code, "slug" => slug, "last_modified" => last_modified, "path" => path }
end
private_class_method :cached_extract
def self.geofabrik_cache_dir(data_dir)
File.join(data_dir, "geofabrik_cache")
end
private_class_method :geofabrik_cache_dir
# Real network implementation of the download_extract seam above —
# streams the extract to dest (some of these are multi-GB — Net::HTTP's
# plain #get buffers the whole response body in memory first, which