Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a84e6ae22f | ||
|
|
920a1d2d36 |
@@ -262,16 +262,35 @@ module Maptool
|
||||
private_class_method :geofabrik_url
|
||||
|
||||
# Real network implementation of the download_extract seam above —
|
||||
# streams the extract to dest and returns its Last-Modified header
|
||||
# streams the extract to dest (some of these are multi-GB — Net::HTTP's
|
||||
# plain #get buffers the whole response body in memory first, which
|
||||
# read_body's block form avoids) and returns its Last-Modified header
|
||||
# (free off the same GET, no separate HEAD needed) for the sidecar
|
||||
# meta / update-check below.
|
||||
def self.download_extract(url, dest)
|
||||
uri = URI(url)
|
||||
response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.get(uri) }
|
||||
raise BuildError, "could not download #{url} (HTTP #{response.code})" unless response.is_a?(Net::HTTPSuccess)
|
||||
# meta / update-check below. Geofabrik 302s (e.g. .../europe/austria
|
||||
# redirecting to wherever actually serves it) — followed manually
|
||||
# since request_get's block form doesn't do this itself.
|
||||
def self.download_extract(url, dest, limit: 5)
|
||||
raise BuildError, "too many redirects downloading #{url}" if limit.zero?
|
||||
|
||||
File.binwrite(dest, response.body)
|
||||
response["last-modified"]
|
||||
uri = URI(url)
|
||||
last_modified = nil
|
||||
redirect_to = nil
|
||||
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
|
||||
http.request_get(uri) do |response|
|
||||
case response
|
||||
when Net::HTTPSuccess
|
||||
last_modified = response["last-modified"]
|
||||
File.open(dest, "wb") { |file| response.read_body { |chunk| file.write(chunk) } }
|
||||
when Net::HTTPRedirection
|
||||
redirect_to = URI.join(url, response["location"]).to_s
|
||||
else
|
||||
raise BuildError, "could not download #{url} (HTTP #{response.code})"
|
||||
end
|
||||
end
|
||||
end
|
||||
return download_extract(redirect_to, dest, limit: limit - 1) if redirect_to
|
||||
|
||||
last_modified
|
||||
end
|
||||
|
||||
# Sidecar recording which Geofabrik sources (country code, slug, and
|
||||
@@ -309,12 +328,16 @@ module Maptool
|
||||
nil
|
||||
end
|
||||
|
||||
def self.fetch_geofabrik_last_modified(url)
|
||||
def self.fetch_geofabrik_last_modified(url, limit: 5)
|
||||
raise BuildError, "too many redirects reaching #{url}" if limit.zero?
|
||||
|
||||
uri = URI(url)
|
||||
response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.head(uri) }
|
||||
raise BuildError, "could not reach #{url}" unless response.is_a?(Net::HTTPSuccess)
|
||||
|
||||
response["last-modified"]
|
||||
case response
|
||||
when Net::HTTPSuccess then response["last-modified"]
|
||||
when Net::HTTPRedirection then fetch_geofabrik_last_modified(URI.join(url, response["location"]).to_s, limit: limit - 1)
|
||||
else raise BuildError, "could not reach #{url}"
|
||||
end
|
||||
end
|
||||
|
||||
def self.offline_meta_path(data_dir)
|
||||
|
||||
Reference in new issue
Block a user