From a84e6ae22f70375e65f782469386cfe403371413 Mon Sep 17 00:00:00 2001 From: Roman Priesol Date: Fri, 11 Sep 2026 20:57:22 +0200 Subject: [PATCH] Follow Geofabrik's 302 redirects on extract download and Last-Modified check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Geofabrik redirects europe/-latest.osm.pbf to a dated filename (confirmed live: .../europe/austria-latest.osm.pbf -> .../europe/austria- 260910.osm.pbf) — a real build hit this immediately. Net::HTTP doesn't follow redirects on its own; both the streaming download and the HEAD-based update-check now do, up to a small limit. --- lib/maptool/pmtiles_builder.rb | 36 ++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/lib/maptool/pmtiles_builder.rb b/lib/maptool/pmtiles_builder.rb index 4354978..9f9cd50 100644 --- a/lib/maptool/pmtiles_builder.rb +++ b/lib/maptool/pmtiles_builder.rb @@ -266,18 +266,30 @@ module Maptool # plain #get buffers the whole response body in memory first, which # read_body's block form avoids) and returns its Last-Modified header # (free off the same GET, no separate HEAD needed) for the sidecar - # meta / update-check below. - def self.download_extract(url, dest) + # meta / update-check below. Geofabrik 302s (e.g. .../europe/austria + # redirecting to wherever actually serves it) — followed manually + # since request_get's block form doesn't do this itself. + def self.download_extract(url, dest, limit: 5) + raise BuildError, "too many redirects downloading #{url}" if limit.zero? + uri = URI(url) last_modified = nil + redirect_to = nil Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http| http.request_get(uri) do |response| - raise BuildError, "could not download #{url} (HTTP #{response.code})" unless response.is_a?(Net::HTTPSuccess) - - last_modified = response["last-modified"] - File.open(dest, "wb") { |file| response.read_body { |chunk| file.write(chunk) } } + case response + when Net::HTTPSuccess + last_modified = response["last-modified"] + File.open(dest, "wb") { |file| response.read_body { |chunk| file.write(chunk) } } + when Net::HTTPRedirection + redirect_to = URI.join(url, response["location"]).to_s + else + raise BuildError, "could not download #{url} (HTTP #{response.code})" + end end end + return download_extract(redirect_to, dest, limit: limit - 1) if redirect_to + last_modified end @@ -316,12 +328,16 @@ module Maptool nil end - def self.fetch_geofabrik_last_modified(url) + def self.fetch_geofabrik_last_modified(url, limit: 5) + raise BuildError, "too many redirects reaching #{url}" if limit.zero? + uri = URI(url) response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.head(uri) } - raise BuildError, "could not reach #{url}" unless response.is_a?(Net::HTTPSuccess) - - response["last-modified"] + case response + when Net::HTTPSuccess then response["last-modified"] + when Net::HTTPRedirection then fetch_geofabrik_last_modified(URI.join(url, response["location"]).to_s, limit: limit - 1) + else raise BuildError, "could not reach #{url}" + end end def self.offline_meta_path(data_dir)