Follow Geofabrik's 302 redirects on extract download and Last-Modified check
Geofabrik redirects europe/<slug>-latest.osm.pbf to a dated filename (confirmed live: .../europe/austria-latest.osm.pbf -> .../europe/austria- 260910.osm.pbf) — a real build hit this immediately. Net::HTTP doesn't follow redirects on its own; both the streaming download and the HEAD-based update-check now do, up to a small limit.
This commit is contained in:
1 parent
920a1d2d36
commit
a84e6ae22f
1 file changed
+24
-8
@@ -266,18 +266,30 @@ module Maptool
|
|||||||
# plain #get buffers the whole response body in memory first, which
|
# plain #get buffers the whole response body in memory first, which
|
||||||
# read_body's block form avoids) and returns its Last-Modified header
|
# read_body's block form avoids) and returns its Last-Modified header
|
||||||
# (free off the same GET, no separate HEAD needed) for the sidecar
|
# (free off the same GET, no separate HEAD needed) for the sidecar
|
||||||
# meta / update-check below.
|
# meta / update-check below. Geofabrik 302s (e.g. .../europe/austria
|
||||||
def self.download_extract(url, dest)
|
# redirecting to wherever actually serves it) — followed manually
|
||||||
|
# since request_get's block form doesn't do this itself.
|
||||||
|
def self.download_extract(url, dest, limit: 5)
|
||||||
|
raise BuildError, "too many redirects downloading #{url}" if limit.zero?
|
||||||
|
|
||||||
uri = URI(url)
|
uri = URI(url)
|
||||||
last_modified = nil
|
last_modified = nil
|
||||||
|
redirect_to = nil
|
||||||
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
|
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
|
||||||
http.request_get(uri) do |response|
|
http.request_get(uri) do |response|
|
||||||
raise BuildError, "could not download #{url} (HTTP #{response.code})" unless response.is_a?(Net::HTTPSuccess)
|
case response
|
||||||
|
when Net::HTTPSuccess
|
||||||
last_modified = response["last-modified"]
|
last_modified = response["last-modified"]
|
||||||
File.open(dest, "wb") { |file| response.read_body { |chunk| file.write(chunk) } }
|
File.open(dest, "wb") { |file| response.read_body { |chunk| file.write(chunk) } }
|
||||||
|
when Net::HTTPRedirection
|
||||||
|
redirect_to = URI.join(url, response["location"]).to_s
|
||||||
|
else
|
||||||
|
raise BuildError, "could not download #{url} (HTTP #{response.code})"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
return download_extract(redirect_to, dest, limit: limit - 1) if redirect_to
|
||||||
|
|
||||||
last_modified
|
last_modified
|
||||||
end
|
end
|
||||||
|
|
||||||
@@ -316,12 +328,16 @@ module Maptool
|
|||||||
nil
|
nil
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.fetch_geofabrik_last_modified(url)
|
def self.fetch_geofabrik_last_modified(url, limit: 5)
|
||||||
|
raise BuildError, "too many redirects reaching #{url}" if limit.zero?
|
||||||
|
|
||||||
uri = URI(url)
|
uri = URI(url)
|
||||||
response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.head(uri) }
|
response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.head(uri) }
|
||||||
raise BuildError, "could not reach #{url}" unless response.is_a?(Net::HTTPSuccess)
|
case response
|
||||||
|
when Net::HTTPSuccess then response["last-modified"]
|
||||||
response["last-modified"]
|
when Net::HTTPRedirection then fetch_geofabrik_last_modified(URI.join(url, response["location"]).to_s, limit: limit - 1)
|
||||||
|
else raise BuildError, "could not reach #{url}"
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.offline_meta_path(data_dir)
|
def self.offline_meta_path(data_dir)
|
||||||
|
|||||||
Reference in new issue
Block a user