diff --git a/lib/maptool/pmtiles_builder.rb b/lib/maptool/pmtiles_builder.rb index d74fd94..b913606 100644 --- a/lib/maptool/pmtiles_builder.rb +++ b/lib/maptool/pmtiles_builder.rb @@ -28,6 +28,13 @@ module Maptool # source's own online maxzoom. TERRARIUM_URL_TEMPLATE = "https://s3.amazonaws.com/elevation-tiles-prod/terrarium/%d/%d/%d.png" TERRAIN_MAX_ZOOM = 15 + # A real bbox at z0..15 is tens of thousands of tiles — serial fetches + # (the original ponytail'd version of this) measured ~0.65s/tile, almost + # all TCP+TLS handshake, no connection reuse: hours for a modest region. + # This many workers, each keeping its own persistent connection (see + # fetch_terrarium_tile), is a big enough win to matter without looking + # like abuse against a public bucket. + CONCURRENT_TILE_FETCHES = 16 OFFLINE_STATUS = { state: "idle" } TERRAIN_STATUS = { state: "idle" } @@ -141,10 +148,11 @@ module Maptool # public Terrarium-encoded PMTiles archive is known to exist to # `extract` from the way offline.pmtiles does above. # - # ponytail: fetches tiles one at a time, serially — fine for a - # park-sized region, slow for a big one. Add a small bounded-concurrency - # fetch pool if regenerating a large area turns out to matter in - # practice. + # fetch_tile.call happens on CONCURRENT_TILE_FETCHES worker threads at + # once (I/O-bound — Ruby's GVL is released while a thread blocks on + # network I/O, so this genuinely overlaps); every SQLite write and + # progress update stays on this thread, draining a results Queue the + # workers feed, so nothing here needs its own locking. def self.write_terrarium_mbtiles(path, south, west, north, east, fetch_tile, progress) db = SQLite3::Database.new(path) db.execute_batch(<<~SQL) @@ -157,29 +165,48 @@ module Maptool end # tile_range per zoom computed once upfront (cheap — no fetching, just - # tile math) and reused below, so progress can report a real total - # from the very first call instead of only once the run finishes. - by_zoom = (0..TERRAIN_MAX_ZOOM).map { |z| [z, tile_range(z, south, west, north, east)] } - total = by_zoom.sum { |_z, tiles| tiles.size } + # tile math), so progress can report a real total from the very first + # call instead of only once the run finishes. + work = (0..TERRAIN_MAX_ZOOM).flat_map { |z| tile_range(z, south, west, north, east).map { |x, y| [z, x, y] } } + total = work.size done = 0 progress.call(done, total) - by_zoom.each do |z, tiles| - tiles.each do |x, y| - bytes = fetch_tile.call(z, x, y) - done += 1 - progress.call(done, total) - next unless bytes + queue = Queue.new + work.each { |item| queue << item } + CONCURRENT_TILE_FETCHES.times { queue << nil } # one stop signal per worker - # MBTiles uses TMS row numbering (0 at the south) — the opposite - # of the XYZ scheme both the Terrarium source and this app's own - # online raster-dem layer use (0 at the north) — flipped here so - # go-pmtiles ends up serving the same tile at the same z/x/y this - # app already requests online. - tms_y = (2**z) - 1 - y - db.execute("INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", [z, x, tms_y, bytes.b]) + results = Queue.new + workers = CONCURRENT_TILE_FETCHES.times.map do + Thread.new do + while (item = queue.pop) + z, x, y = item + results << [z, x, y, fetch_tile.call(z, x, y)] + end + ensure + # Only set when fetch_tile is fetch_terrarium_tile (a test fake + # has no reason to touch this) — closes this worker's persistent + # connection rather than leaving the socket for GC to eventually + # notice. + Thread.current[:maptool_terrarium_http]&.finish end end + + total.times do + z, x, y, bytes = results.pop + done += 1 + progress.call(done, total) + next unless bytes + + # MBTiles uses TMS row numbering (0 at the south) — the opposite of + # the XYZ scheme both the Terrarium source and this app's own online + # raster-dem layer use (0 at the north) — flipped here so go-pmtiles + # ends up serving the same tile at the same z/x/y this app already + # requests online. + tms_y = (2**z) - 1 - y + db.execute("INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", [z, x, tms_y, bytes.b]) + end + workers.each(&:join) ensure db&.close end @@ -204,8 +231,14 @@ module Maptool end private_class_method :lat_to_tile_y + # One persistent connection per calling Thread (each of + # write_terrarium_mbtiles' worker threads calls this many times) instead + # of Net::HTTP.get_response's usual "new TCP+TLS handshake per call" — + # that handshake was most of the ~0.65s/tile this used to cost. def self.fetch_terrarium_tile(z, x, y) - response = Net::HTTP.get_response(URI(format(TERRARIUM_URL_TEMPLATE, z: z, x: x, y: y))) + uri = URI(format(TERRARIUM_URL_TEMPLATE, z: z, x: x, y: y)) + http = (Thread.current[:maptool_terrarium_http] ||= Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https")) + response = http.get(uri.request_uri) response.is_a?(Net::HTTPSuccess) ? response.body : nil rescue StandardError nil diff --git a/test/test_pmtiles_builder.rb b/test/test_pmtiles_builder.rb index fda9dab..e6b9798 100644 --- a/test/test_pmtiles_builder.rb +++ b/test/test_pmtiles_builder.rb @@ -119,27 +119,34 @@ class PmtilesBuilderTest < Minitest::Test assert_equal status[:tiles_total], status[:tiles_done] end - # fetch_tile blocks after the very first call so the "running" progress - # can be observed mid-flight, deterministically, rather than only ever - # seeing the final done/total once the (very fast, faked) build finishes. + # Every fetch_tile call blocks on a gate until explicitly told not to — + # with CONCURRENT_TILE_FETCHES workers pulling from a shared queue, that + # holds the whole pool at "in flight, none finished yet" deterministically + # (a big enough bbox that z0..15 yields far more tiles than worker slots, + # so most of the job is still queued up behind the blocked pool). def test_build_terrain_reports_progress_while_running ENV["PMTILES_BIN"] = @pmtiles_ok + concurrency = Maptool::PmtilesBuilder::CONCURRENT_TILE_FETCHES gate = Queue.new - first_call = true + unblocked = false fetch_tile = lambda do |_z, _x, _y| - gate.pop if first_call - first_call = false + gate.pop unless unblocked "bytes" end - thread = Maptool::PmtilesBuilder.build_terrain([48.140, 17.100, 48.141, 17.101], data_dir: @data_dir, fetch_tile: fetch_tile) + thread = Maptool::PmtilesBuilder.build_terrain([47.5, 16.5, 48.5, 17.5], data_dir: @data_dir, fetch_tile: fetch_tile) - 20.times { break if Maptool::PmtilesBuilder.terrain_status[:tiles_total]; sleep 0.01 } + # Let exactly one batch through (fills every worker's first call) then + # check before releasing the rest — done should be capped at the pool + # size, nowhere near the real total. + concurrency.times { gate.push(nil) } + sleep 0.05 status = Maptool::PmtilesBuilder.terrain_status assert_equal "running", status[:state] - assert_operator status[:tiles_total], :>, 0 - assert_equal 0, status[:tiles_done] + assert_operator status[:tiles_total], :>, concurrency + assert_operator status[:tiles_done], :<, status[:tiles_total] ensure - gate.push(nil) + unblocked = true + concurrency.times { gate.push(nil) } # release whatever's still blocked in an in-progress gate.pop thread&.join end