From 920a1d2d3670421f5d699335e26f35cb48063e23 Mon Sep 17 00:00:00 2001 From: Roman Priesol Date: Fri, 11 Sep 2026 20:56:06 +0200 Subject: [PATCH] Stream Geofabrik extract downloads instead of buffering in memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Net::HTTP's plain #get loads the whole response body before returning — fine for the small JSON manifests elsewhere in this file, but a real risk for a multi-GB country .osm.pbf (Germany, Russia, ...). Streams to disk via read_body instead. --- lib/maptool/pmtiles_builder.rb | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/maptool/pmtiles_builder.rb b/lib/maptool/pmtiles_builder.rb index 9aa5319..4354978 100644 --- a/lib/maptool/pmtiles_builder.rb +++ b/lib/maptool/pmtiles_builder.rb @@ -262,16 +262,23 @@ module Maptool private_class_method :geofabrik_url # Real network implementation of the download_extract seam above — - # streams the extract to dest and returns its Last-Modified header + # streams the extract to dest (some of these are multi-GB — Net::HTTP's + # plain #get buffers the whole response body in memory first, which + # read_body's block form avoids) and returns its Last-Modified header # (free off the same GET, no separate HEAD needed) for the sidecar # meta / update-check below. def self.download_extract(url, dest) uri = URI(url) - response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.get(uri) } - raise BuildError, "could not download #{url} (HTTP #{response.code})" unless response.is_a?(Net::HTTPSuccess) + last_modified = nil + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http| + http.request_get(uri) do |response| + raise BuildError, "could not download #{url} (HTTP #{response.code})" unless response.is_a?(Net::HTTPSuccess) - File.binwrite(dest, response.body) - response["last-modified"] + last_modified = response["last-modified"] + File.open(dest, "wb") { |file| response.read_body { |chunk| file.write(chunk) } } + end + end + last_modified end # Sidecar recording which Geofabrik sources (country code, slug, and