Stream Geofabrik extract downloads instead of buffering in memory
Net::HTTP's plain #get loads the whole response body before returning — fine for the small JSON manifests elsewhere in this file, but a real risk for a multi-GB country .osm.pbf (Germany, Russia, ...). Streams to disk via read_body instead.
This commit is contained in:
1 parent
d08ed41185
commit
920a1d2d36
1 file changed
+11
-4
@@ -262,16 +262,23 @@ module Maptool
|
||||
private_class_method :geofabrik_url
|
||||
|
||||
# Real network implementation of the download_extract seam above —
|
||||
# streams the extract to dest and returns its Last-Modified header
|
||||
# streams the extract to dest (some of these are multi-GB — Net::HTTP's
|
||||
# plain #get buffers the whole response body in memory first, which
|
||||
# read_body's block form avoids) and returns its Last-Modified header
|
||||
# (free off the same GET, no separate HEAD needed) for the sidecar
|
||||
# meta / update-check below.
|
||||
def self.download_extract(url, dest)
|
||||
uri = URI(url)
|
||||
response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.get(uri) }
|
||||
last_modified = nil
|
||||
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
|
||||
http.request_get(uri) do |response|
|
||||
raise BuildError, "could not download #{url} (HTTP #{response.code})" unless response.is_a?(Net::HTTPSuccess)
|
||||
|
||||
File.binwrite(dest, response.body)
|
||||
response["last-modified"]
|
||||
last_modified = response["last-modified"]
|
||||
File.open(dest, "wb") { |file| response.read_body { |chunk| file.write(chunk) } }
|
||||
end
|
||||
end
|
||||
last_modified
|
||||
end
|
||||
|
||||
# Sidecar recording which Geofabrik sources (country code, slug, and
|
||||
|
||||
Reference in new issue
Block a user