Compare commits

...
2 Commits
Author SHA1 Message Date
randy a84e6ae22f Follow Geofabrik's 302 redirects on extract download and Last-Modified check
CI / deploy (push) Successful in 16s
CI / check (push) Successful in 29s
Geofabrik redirects europe/<slug>-latest.osm.pbf to a dated filename
(confirmed live: .../europe/austria-latest.osm.pbf -> .../europe/austria-
260910.osm.pbf) — a real build hit this immediately. Net::HTTP doesn't
follow redirects on its own; both the streaming download and the HEAD-based
update-check now do, up to a small limit.
2026-09-11 20:57:22 +02:00
randy 920a1d2d36 Stream Geofabrik extract downloads instead of buffering in memory
Net::HTTP's plain #get loads the whole response body before returning —
fine for the small JSON manifests elsewhere in this file, but a real risk
for a multi-GB country .osm.pbf (Germany, Russia, ...). Streams to disk
via read_body instead.
2026-09-11 20:56:06 +02:00
+35 -12
View File
@@ -262,16 +262,35 @@ module Maptool
private_class_method :geofabrik_url private_class_method :geofabrik_url
# Real network implementation of the download_extract seam above — # Real network implementation of the download_extract seam above —
# streams the extract to dest and returns its Last-Modified header # streams the extract to dest (some of these are multi-GB — Net::HTTP's
# plain #get buffers the whole response body in memory first, which
# read_body's block form avoids) and returns its Last-Modified header
# (free off the same GET, no separate HEAD needed) for the sidecar # (free off the same GET, no separate HEAD needed) for the sidecar
# meta / update-check below. # meta / update-check below. Geofabrik 302s (e.g. .../europe/austria
def self.download_extract(url, dest) # redirecting to wherever actually serves it) — followed manually
uri = URI(url) # since request_get's block form doesn't do this itself.
response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.get(uri) } def self.download_extract(url, dest, limit: 5)
raise BuildError, "could not download #{url} (HTTP #{response.code})" unless response.is_a?(Net::HTTPSuccess) raise BuildError, "too many redirects downloading #{url}" if limit.zero?
File.binwrite(dest, response.body) uri = URI(url)
response["last-modified"] last_modified = nil
redirect_to = nil
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
http.request_get(uri) do |response|
case response
when Net::HTTPSuccess
last_modified = response["last-modified"]
File.open(dest, "wb") { |file| response.read_body { |chunk| file.write(chunk) } }
when Net::HTTPRedirection
redirect_to = URI.join(url, response["location"]).to_s
else
raise BuildError, "could not download #{url} (HTTP #{response.code})"
end
end
end
return download_extract(redirect_to, dest, limit: limit - 1) if redirect_to
last_modified
end end
# Sidecar recording which Geofabrik sources (country code, slug, and # Sidecar recording which Geofabrik sources (country code, slug, and
@@ -309,12 +328,16 @@ module Maptool
nil nil
end end
def self.fetch_geofabrik_last_modified(url) def self.fetch_geofabrik_last_modified(url, limit: 5)
raise BuildError, "too many redirects reaching #{url}" if limit.zero?
uri = URI(url) uri = URI(url)
response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.head(uri) } response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.head(uri) }
raise BuildError, "could not reach #{url}" unless response.is_a?(Net::HTTPSuccess) case response
when Net::HTTPSuccess then response["last-modified"]
response["last-modified"] when Net::HTTPRedirection then fetch_geofabrik_last_modified(URI.join(url, response["location"]).to_s, limit: limit - 1)
else raise BuildError, "could not reach #{url}"
end
end end
def self.offline_meta_path(data_dir) def self.offline_meta_path(data_dir)