Cache per-country Geofabrik extracts so Apply doesn't redownload everything
Every region-selection change re-downloaded every selected country's .osm.pbf from scratch, even ones already fetched by a prior build — painful for multi-GB countries. Extracts now persist in data/geofabrik_cache/, keyed by country, and are only re-fetched when Geofabrik's Last-Modified for that country actually moved on (checked with a cheap HEAD) or the file isn't cached yet.
This commit is contained in:
1 parent
077b44ece1
commit
007dee7341
4 files changed
+131
-17
No files matched your search
@@ -179,17 +179,17 @@ module Maptool
|
||||
# finishes — raises BuildError (the caller turns this into an error
|
||||
# response) if nothing's selected yet, or one's already running.
|
||||
#
|
||||
# download_extract exists as an injectable seam so tests can exercise
|
||||
# the rest of this (scratch dir, osmium/planetiler invocation, atomic
|
||||
# rename) without a real network call — same reasoning test_helper.rb
|
||||
# already gives for keeping /api/plugins route tests off the real
|
||||
# Overpass API.
|
||||
def self.build_offline(data_dir:, download_extract: method(:download_extract))
|
||||
# download_extract/fetch_last_modified exist as injectable seams so
|
||||
# tests can exercise the rest of this (scratch dir, osmium/planetiler
|
||||
# invocation, atomic rename, the per-country cache below) without a
|
||||
# real network call — same reasoning test_helper.rb already gives for
|
||||
# keeping /api/plugins route tests off the real Overpass API.
|
||||
def self.build_offline(data_dir:, download_extract: method(:download_extract), fetch_last_modified: method(:fetch_geofabrik_last_modified))
|
||||
codes = selected_regions(data_dir)
|
||||
raise BuildError, "select at least one region first" if codes.empty?
|
||||
|
||||
begin_run!(OFFLINE_MUTEX, OFFLINE_STATUS)
|
||||
Thread.new { run_job(OFFLINE_MUTEX, OFFLINE_STATUS) { extract_offline(codes, data_dir, download_extract) } }
|
||||
Thread.new { run_job(OFFLINE_MUTEX, OFFLINE_STATUS) { extract_offline(codes, data_dir, download_extract, fetch_last_modified) } }
|
||||
end
|
||||
|
||||
# fetch_manifest: same injectable-seam reasoning as download_extract
|
||||
@@ -228,13 +228,11 @@ module Maptool
|
||||
# tiles.openfreemap.org's Liberty style already expects, so
|
||||
# public/js/map.js's offlineStyle() can reuse ONLINE_STYLE_PATCH
|
||||
# unchanged instead of a separate protomaps-themes-base palette.
|
||||
def self.extract_offline(codes, data_dir, download_extract)
|
||||
def self.extract_offline(codes, data_dir, download_extract, fetch_last_modified)
|
||||
scratch_dir = Dir.mktmpdir("maptool-offline")
|
||||
sources = codes.map do |code|
|
||||
slug = GEOFABRIK_SLUGS.fetch(code) { raise BuildError, "no Geofabrik source known for region #{code}" }
|
||||
dest = File.join(scratch_dir, "#{code}.osm.pbf")
|
||||
last_modified = download_extract.call(geofabrik_url(slug), dest)
|
||||
{ "code" => code, "slug" => slug, "last_modified" => last_modified, "path" => dest }
|
||||
cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
|
||||
end
|
||||
|
||||
osm_path =
|
||||
@@ -261,6 +259,41 @@ module Maptool
|
||||
end
|
||||
private_class_method :geofabrik_url
|
||||
|
||||
# Per-country .osm.pbf cache, kept across builds under data_dir (not
|
||||
# scratch_dir, which is wiped every run) — changing the region
|
||||
# selection re-downloads only the countries that are new to it, and
|
||||
# re-running the same selection re-downloads nothing unless Geofabrik's
|
||||
# own Last-Modified for that country moved on. A stamp file next to
|
||||
# each cached extract records the Last-Modified it was fetched with;
|
||||
# a HEAD check (cheap) decides whether the multi-GB GET (not cheap) is
|
||||
# skippable.
|
||||
def self.cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
|
||||
cache_dir = geofabrik_cache_dir(data_dir)
|
||||
FileUtils.mkdir_p(cache_dir)
|
||||
path = File.join(cache_dir, "#{code}.osm.pbf")
|
||||
stamp_path = "#{path}.last_modified"
|
||||
url = geofabrik_url(slug)
|
||||
|
||||
if File.exist?(path) && File.exist?(stamp_path)
|
||||
current = begin
|
||||
fetch_last_modified.call(url)
|
||||
rescue BuildError
|
||||
nil
|
||||
end
|
||||
return { "code" => code, "slug" => slug, "last_modified" => current, "path" => path } if current && current == File.read(stamp_path)
|
||||
end
|
||||
|
||||
last_modified = download_extract.call(url, path)
|
||||
File.write(stamp_path, last_modified.to_s)
|
||||
{ "code" => code, "slug" => slug, "last_modified" => last_modified, "path" => path }
|
||||
end
|
||||
private_class_method :cached_extract
|
||||
|
||||
def self.geofabrik_cache_dir(data_dir)
|
||||
File.join(data_dir, "geofabrik_cache")
|
||||
end
|
||||
private_class_method :geofabrik_cache_dir
|
||||
|
||||
# Real network implementation of the download_extract seam above —
|
||||
# streams the extract to dest (some of these are multi-GB — Net::HTTP's
|
||||
# plain #get buffers the whole response body in memory first, which
|
||||
|
||||
Reference in new issue
Block a user