maptool/lib/maptool/pmtiles_builder.rb

require "open3"
require "net/http"
require "uri"
require "json"
require "tmpdir"
require "fileutils"

module Maptool
  # Regenerates data/offline.pmtiles and data/terrain.pmtiles for the
  # currently *selected* regions (data/offline_regions.json, an array of
  # Natural Earth iso_a2 country codes — see available_regions/
  # selected_regions/set_selected_regions below). No job queue: each build
  # runs on its own background Thread and reports into a Mutex-guarded
  # status Hash a route can poll, the same shape app.rb's own GNSS poller
  # (GNSS_LATEST/GNSS_LATEST_MUTEX) already uses.
  module PmtilesBuilder
    class BuildError < StandardError; end

    # Natural Earth (public domain, no attribution required) 110m admin-0
    # countries, filtered to Europe and trimmed to name/iso_a2/iso_a3 —
    # see the plan this was built from for the exact filter. 110m is
    # deliberately coarse (a km or so of border precision, not
    # survey-grade) — fine for "which country to grab," a same-schema swap
    # to Natural Earth's 50m/10m cut if that ever needs to be tighter.
    REGIONS_FILE = File.join(__dir__, "data", "europe_countries.geojson")

    # Every iso_a2 code in REGIONS_FILE mapped to its Geofabrik download
    # slug (https://download.geofabrik.de/<slug>-latest.osm.pbf) — Geofabrik
    # names regions, not ISO codes, and doesn't always split the way a
    # country border does, so this table exists rather than deriving the
    # slug from the country name. Two known quirks worth knowing about,
    # not worth working around:
    #  - RU has no "europe/" prefix (Russia is its own top-level index,
    #    and unlike every other entry here is a single ~4GB file covering
    #    the whole country rather than a European-only slice).
    #  - GB's "great-britain" slug is England/Scotland/Wales only —
    #    Northern Ireland lives in "ireland-and-northern-ireland" instead
    #    (bundled with IE, not GB). Selecting GB alone won't include NI.
    GEOFABRIK_SLUGS = {
      "AL" => "europe/albania",
      "AT" => "europe/austria",
      "BY" => "europe/belarus",
      "BE" => "europe/belgium",
      "BA" => "europe/bosnia-herzegovina",
      "BG" => "europe/bulgaria",
      "HR" => "europe/croatia",
      "CZ" => "europe/czech-republic",
      "DK" => "europe/denmark",
      "EE" => "europe/estonia",
      "FI" => "europe/finland",
      "FR" => "europe/france",
      "DE" => "europe/germany",
      "GR" => "europe/greece",
      "HU" => "europe/hungary",
      "IS" => "europe/iceland",
      "IE" => "europe/ireland-and-northern-ireland",
      "IT" => "europe/italy",
      "XK" => "europe/kosovo",
      "LV" => "europe/latvia",
      "LT" => "europe/lithuania",
      "LU" => "europe/luxembourg",
      "MD" => "europe/moldova",
      "ME" => "europe/montenegro",
      "NL" => "europe/netherlands",
      "MK" => "europe/macedonia",
      "NO" => "europe/norway",
      "PL" => "europe/poland",
      "PT" => "europe/portugal",
      "RO" => "europe/romania",
      "RU" => "russia",
      "RS" => "europe/serbia",
      "SK" => "europe/slovakia",
      "SI" => "europe/slovenia",
      "ES" => "europe/spain",
      "SE" => "europe/sweden",
      "CH" => "europe/switzerland",
      "UA" => "europe/ukraine",
      "GB" => "europe/great-britain",
    }.freeze

    # Mapterhorn (a Protomaps-affiliated project) publishes Terrarium-
    # encoded terrain as pre-built PMTiles archives: one global
    # planet.pmtiles (z0-12) plus ~450 regional chunks (z13 up to a
    # per-region max, 13-18) — extract_offline below no longer touches
    # Protomaps at all (see GEOFABRIK_SLUGS above), but terrain stays on
    # this same "extract only the region's byte ranges from a remote
    # archive" story via go-pmtiles, unrelated to the basemap swap.
    MAPTERHORN_MANIFEST_URL = "https://download.mapterhorn.com/download_urls.json"
    MAPTERHORN_PLANET_NAME = "planet.pmtiles"
    # Matches the online hillshade layer's own cap (public/js/overlays.js's
    # ONLINE_TERRAIN_TILES source, maxzoom: 15) — applied uniformly even
    # though some Mapterhorn chunks natively offer more, so the offline
    # experience doesn't vary in detail from one region to the next.
    TERRAIN_MAX_ZOOM = 15

    OFFLINE_STATUS = { state: "idle" }
    TERRAIN_STATUS = { state: "idle" }
    OFFLINE_MUTEX = Mutex.new
    TERRAIN_MUTEX = Mutex.new

    def self.offline_status
      OFFLINE_MUTEX.synchronize { OFFLINE_STATUS.dup }
    end

    def self.terrain_status
      TERRAIN_MUTEX.synchronize { TERRAIN_STATUS.dup }
    end

    # Test-only: the status hashes are process-wide state (same caveat as
    # GPX_STORE/LIVETRACK_STORE in app.rb), so the suite needs a way to
    # reset them between tests the way AppTestHelper#teardown already does
    # for those.
    def self.reset!
      OFFLINE_MUTEX.synchronize { OFFLINE_STATUS.replace(state: "idle") }
      TERRAIN_MUTEX.synchronize { TERRAIN_STATUS.replace(state: "idle") }
    end

    # --- region selection ---

    def self.available_regions
      region_features.map { |f| { code: f["properties"]["iso_a2"], name: f["properties"]["name"] } }.sort_by { |r| r[:name] }
    end

    def self.selected_regions(data_dir)
      path = selected_regions_path(data_dir)
      return [] unless File.exist?(path)

      JSON.parse(File.read(path))
    rescue JSON::ParserError
      []
    end

    def self.set_selected_regions(data_dir, codes)
      unknown = codes - available_regions.map { |r| r[:code] }
      raise BuildError, "unknown region code(s): #{unknown.join(", ")}" unless unknown.empty?

      File.write(selected_regions_path(data_dir), JSON.generate(codes))
    end

    def self.selected_regions_path(data_dir)
      File.join(data_dir, "offline_regions.json")
    end
    private_class_method :selected_regions_path

    def self.region_features
      @region_features ||= JSON.parse(File.read(REGIONS_FILE))["features"]
    end
    private_class_method :region_features

    # Still used by terrain (below) to compute the aggregate bbox Mapterhorn
    # chunk intersection needs — offline.pmtiles no longer needs this at
    # all, since each Geofabrik country file is already clipped to that
    # country's real border (more accurate than this 110m polygon, not
    # less), so there's no separate "--region=" precision step for it
    # anymore.
    def self.region_bbox(codes)
      selected = region_features.select { |f| codes.include?(f["properties"]["iso_a2"]) }
      raise BuildError, "no matching regions for #{codes.join(", ")}" if selected.empty?

      coords = selected.flat_map { |f| ring_coordinates(f["geometry"]) }
      lons = coords.map { |lng, _lat| lng }
      lats = coords.map { |_lng, lat| lat }
      [lats.min, lons.min, lats.max, lons.max]
    end
    private_class_method :region_bbox

    def self.ring_coordinates(geometry)
      case geometry["type"]
      when "Polygon" then geometry["coordinates"].flatten(1)
      when "MultiPolygon" then geometry["coordinates"].flatten(2)
      else []
      end
    end
    private_class_method :ring_coordinates

    # --- builds ---

    # Returns once the background Thread has started, not once the build
    # finishes — raises BuildError (the caller turns this into an error
    # response) if nothing's selected yet, or one's already running.
    #
    # download_extract/fetch_last_modified exist as injectable seams so
    # tests can exercise the rest of this (scratch dir, osmium/planetiler
    # invocation, atomic rename, the per-country cache below) without a
    # real network call — same reasoning test_helper.rb already gives for
    # keeping /api/plugins route tests off the real Overpass API.
    def self.build_offline(data_dir:, download_extract: method(:download_extract), fetch_last_modified: method(:fetch_geofabrik_last_modified))
      codes = selected_regions(data_dir)
      raise BuildError, "select at least one region first" if codes.empty?

      begin_run!(OFFLINE_MUTEX, OFFLINE_STATUS)
      Thread.new { run_job(OFFLINE_MUTEX, OFFLINE_STATUS) { extract_offline(codes, data_dir, download_extract, fetch_last_modified) } }
    end

    # fetch_manifest: same injectable-seam reasoning as download_extract
    # above, for the one Mapterhorn manifest lookup this needs.
    def self.build_terrain(data_dir:, fetch_manifest: method(:fetch_mapterhorn_manifest))
      codes = selected_regions(data_dir)
      raise BuildError, "select at least one region first" if codes.empty?

      begin_run!(TERRAIN_MUTEX, TERRAIN_STATUS)
      Thread.new { run_job(TERRAIN_MUTEX, TERRAIN_STATUS) { build_terrain_pmtiles(codes, data_dir, fetch_manifest) } }
    end

    def self.begin_run!(mutex, status)
      mutex.synchronize do
        raise BuildError, "a build is already running" if status[:state] == "running"
        status.replace(state: "running", started_at: Time.now.utc.iso8601)
      end
    end
    private_class_method :begin_run!

    def self.run_job(mutex, status)
      yield
      mutex.synchronize { status.merge!(state: "done", finished_at: Time.now.utc.iso8601).delete(:message) }
    rescue => e
      mutex.synchronize { status.merge!(state: "error", finished_at: Time.now.utc.iso8601, message: e.message) }
    end
    private_class_method :run_job

    # --- offline.pmtiles ---

    # One Geofabrik .osm.pbf per selected country (already clipped to that
    # country's real border), osmium-merged into one file if more than one
    # was selected (skipped entirely for a single country — nothing to
    # merge), then tiled by Planetiler using its default OpenMapTiles
    # profile straight to PMTiles — the same vector schema
    # tiles.openfreemap.org's Liberty style already expects, so
    # public/js/map.js's offlineStyle() can reuse ONLINE_STYLE_PATCH
    # unchanged instead of a separate protomaps-themes-base palette.
    def self.extract_offline(codes, data_dir, download_extract, fetch_last_modified)
      scratch_dir = Dir.mktmpdir("maptool-offline")
      sources = codes.map do |code|
        slug = GEOFABRIK_SLUGS.fetch(code) { raise BuildError, "no Geofabrik source known for region #{code}" }
        cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
      end

      osm_path =
        if sources.size == 1
          sources.first["path"]
        else
          merged_path = File.join(scratch_dir, "merged.osm.pbf")
          run_osmium!("merge", *sources.map { |s| s["path"] }, "-o", merged_path)
          merged_path
        end

      tmp_path = File.join(data_dir, "offline.pmtiles.tmp")
      run_planetiler!("--osm-path=#{osm_path}", "--output=#{tmp_path}")
      File.rename(tmp_path, File.join(data_dir, "offline.pmtiles"))
      File.write(offline_meta_path(data_dir), JSON.generate(sources: sources.map { |s| s.slice("code", "slug", "last_modified") }))
    ensure
      FileUtils.remove_entry(scratch_dir) if scratch_dir && Dir.exist?(scratch_dir)
      File.delete(tmp_path) if tmp_path && File.exist?(tmp_path)
    end
    private_class_method :extract_offline

    def self.geofabrik_url(slug)
      "https://download.geofabrik.de/#{slug}-latest.osm.pbf"
    end
    private_class_method :geofabrik_url

    # Per-country .osm.pbf cache, kept across builds under data_dir (not
    # scratch_dir, which is wiped every run) — changing the region
    # selection re-downloads only the countries that are new to it, and
    # re-running the same selection re-downloads nothing unless Geofabrik's
    # own Last-Modified for that country moved on. A stamp file next to
    # each cached extract records the Last-Modified it was fetched with;
    # a HEAD check (cheap) decides whether the multi-GB GET (not cheap) is
    # skippable.
    def self.cached_extract(code, slug, data_dir, download_extract, fetch_last_modified)
      cache_dir = geofabrik_cache_dir(data_dir)
      FileUtils.mkdir_p(cache_dir)
      path = File.join(cache_dir, "#{code}.osm.pbf")
      stamp_path = "#{path}.last_modified"
      url = geofabrik_url(slug)

      if File.exist?(path) && File.exist?(stamp_path)
        current = begin
          fetch_last_modified.call(url)
        rescue BuildError
          nil
        end
        return { "code" => code, "slug" => slug, "last_modified" => current, "path" => path } if current && current == File.read(stamp_path)
      end

      last_modified = download_extract.call(url, path)
      File.write(stamp_path, last_modified.to_s)
      { "code" => code, "slug" => slug, "last_modified" => last_modified, "path" => path }
    end
    private_class_method :cached_extract

    def self.geofabrik_cache_dir(data_dir)
      File.join(data_dir, "geofabrik_cache")
    end
    private_class_method :geofabrik_cache_dir

    # Real network implementation of the download_extract seam above —
    # streams the extract to dest (some of these are multi-GB — Net::HTTP's
    # plain #get buffers the whole response body in memory first, which
    # read_body's block form avoids) and returns its Last-Modified header
    # (free off the same GET, no separate HEAD needed) for the sidecar
    # meta / update-check below. Geofabrik 302s (e.g. .../europe/austria
    # redirecting to wherever actually serves it) — followed manually
    # since request_get's block form doesn't do this itself.
    def self.download_extract(url, dest, limit: 5)
      raise BuildError, "too many redirects downloading #{url}" if limit.zero?

      uri = URI(url)
      last_modified = nil
      redirect_to = nil
      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
        http.request_get(uri) do |response|
          case response
          when Net::HTTPSuccess
            last_modified = response["last-modified"]
            File.open(dest, "wb") { |file| response.read_body { |chunk| file.write(chunk) } }
          when Net::HTTPRedirection
            redirect_to = URI.join(url, response["location"]).to_s
          else
            raise BuildError, "could not download #{url} (HTTP #{response.code})"
          end
        end
      end
      return download_extract(redirect_to, dest, limit: limit - 1) if redirect_to

      last_modified
    end

    # Sidecar recording which Geofabrik sources (country code, slug, and
    # that request's Last-Modified) the on-disk offline.pmtiles was
    # actually built from — the file itself carries no such record. nil if
    # never built (or built before this existed).
    def self.offline_meta(data_dir)
      read_json(offline_meta_path(data_dir))
    end

    # Compares each currently-selected country's live Geofabrik
    # Last-Modified against what's recorded, without downloading anything
    # — nil (not true/false) if there's nothing to compare yet (no local
    # build, nothing selected) or a check couldn't be completed, so the
    # caller can tell "don't know" apart from "no". Also naturally reports
    # "yes" if the selection changed since the last build (a country now
    # relevant that wasn't recorded, or vice versa) — same reasoning
    # terrain_update_available? below already uses.
    def self.offline_update_available?(data_dir, fetch_last_modified: method(:fetch_geofabrik_last_modified))
      meta = offline_meta(data_dir)
      return nil unless meta

      codes = selected_regions(data_dir)
      return nil if codes.empty?

      current = codes.map do |code|
        slug = GEOFABRIK_SLUGS[code]
        return nil unless slug

        { "code" => code, "last_modified" => fetch_last_modified.call(geofabrik_url(slug)) }
      end.sort_by { |s| s["code"] }
      recorded = (meta["sources"] || []).map { |s| s.slice("code", "last_modified") }.sort_by { |s| s["code"] }
      current != recorded
    rescue BuildError
      nil
    end

    def self.fetch_geofabrik_last_modified(url, limit: 5)
      raise BuildError, "too many redirects reaching #{url}" if limit.zero?

      uri = URI(url)
      response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") { |http| http.head(uri) }
      case response
      when Net::HTTPSuccess then response["last-modified"]
      when Net::HTTPRedirection then fetch_geofabrik_last_modified(URI.join(url, response["location"]).to_s, limit: limit - 1)
      else raise BuildError, "could not reach #{url}"
      end
    end

    def self.offline_meta_path(data_dir)
      File.join(data_dir, "offline.pmtiles.meta.json")
    end
    private_class_method :offline_meta_path

    # --- terrain.pmtiles ---

    # planet.pmtiles (z0-12, always) plus whichever regional chunks
    # intersect the selection's aggregate bbox (z13+, zero or more) are
    # each `extract`ed on their own using the *same* --region= file, then
    # combined with `pmtiles merge` if there was more than one. Always
    # safe to merge regardless of how many countries are selected or
    # whether they're adjacent: each Mapterhorn chunk is its own
    # pre-partitioned, non-overlapping source grid cell, so two different
    # chunks' extracts can never share a tile id — the only thing `merge`
    # refuses (see pmtiles/merge.go's own disjointness check). What would
    # NOT be safe is extracting the *same* source once per country and
    # merging those — adjacent countries' region-extracts would share
    # border tiles (extraction copies whole tiles, it doesn't clip vector
    # content to the polygon) — so this always does one extract per
    # *source archive*, never one per country.
    def self.build_terrain_pmtiles(codes, data_dir, fetch_manifest)
      items = fetch_manifest.call
      planet = items.find { |item| item["name"] == MAPTERHORN_PLANET_NAME }
      raise BuildError, "mapterhorn manifest is missing #{MAPTERHORN_PLANET_NAME}" unless planet

      scratch_dir = Dir.mktmpdir("maptool-terrain")
      south, west, north, east = region_bbox(codes)
      region_path = File.join(scratch_dir, "region.geojson")
      selected = region_features.select { |f| codes.include?(f["properties"]["iso_a2"]) }
      File.write(region_path, JSON.generate(type: "FeatureCollection", features: selected))

      chunks = items.select { |item| item["name"] != MAPTERHORN_PLANET_NAME && bbox_intersects?(item, south, west, north, east) }

      tmp_path = File.join(data_dir, "terrain.pmtiles.tmp")
      extracts = [planet, *chunks].each_with_index.map do |item, i|
        extract_path = File.join(scratch_dir, "extract-#{i}.pmtiles")
        run_pmtiles!("extract", item["url"], extract_path, "--region=#{region_path}", "--maxzoom=#{TERRAIN_MAX_ZOOM}")
        extract_path
      end

      if extracts.size == 1
        File.rename(extracts.first, tmp_path)
      else
        run_pmtiles!("merge", *extracts, tmp_path)
      end
      File.rename(tmp_path, File.join(data_dir, "terrain.pmtiles"))
      File.write(terrain_meta_path(data_dir), JSON.generate(sources: [planet, *chunks].map { |i| { name: i["name"], md5sum: i["md5sum"] } }))
    ensure
      FileUtils.remove_entry(scratch_dir) if scratch_dir && Dir.exist?(scratch_dir)
      File.delete(tmp_path) if tmp_path && File.exist?(tmp_path)
    end
    private_class_method :build_terrain_pmtiles

    def self.bbox_intersects?(item, south, west, north, east)
      item["min_lon"] < east && item["max_lon"] > west && item["min_lat"] < north && item["max_lat"] > south
    end
    private_class_method :bbox_intersects?

    def self.fetch_mapterhorn_manifest
      response = Net::HTTP.get_response(URI(MAPTERHORN_MANIFEST_URL))
      raise BuildError, "could not reach #{MAPTERHORN_MANIFEST_URL}" unless response.is_a?(Net::HTTPSuccess)

      JSON.parse(response.body).fetch("items")
    rescue JSON::ParserError
      raise BuildError, "mapterhorn manifest was not valid JSON"
    end
    private_class_method :fetch_mapterhorn_manifest

    # Sidecar recording which Mapterhorn sources (name + md5sum) the on-disk
    # terrain.pmtiles was actually extracted+merged from. nil if never built.
    def self.terrain_meta(data_dir)
      read_json(terrain_meta_path(data_dir))
    end

    # Same "don't know" vs. "no" distinction as offline_update_available?
    # above. Recomputes which sources *would* be selected for the current
    # selection (same manifest + bbox_intersects? this build itself uses)
    # and compares their md5sums against what's recorded — so this also
    # correctly says "yes" if the selection changed since the last build
    # (a chunk now relevant that wasn't recorded), not just if Mapterhorn's
    # own data moved on.
    def self.terrain_update_available?(data_dir, fetch_manifest: method(:fetch_mapterhorn_manifest))
      meta = terrain_meta(data_dir)
      return nil unless meta

      codes = selected_regions(data_dir)
      return nil if codes.empty?

      items = fetch_manifest.call
      planet = items.find { |item| item["name"] == MAPTERHORN_PLANET_NAME }
      return nil unless planet

      south, west, north, east = region_bbox(codes)
      chunks = items.select { |item| item["name"] != MAPTERHORN_PLANET_NAME && bbox_intersects?(item, south, west, north, east) }
      current = [planet, *chunks].map { |i| { "name" => i["name"], "md5sum" => i["md5sum"] } }.sort_by { |s| s["name"] }
      current != (meta["sources"] || []).sort_by { |s| s["name"] }
    rescue BuildError
      nil
    end

    def self.terrain_meta_path(data_dir)
      File.join(data_dir, "terrain.pmtiles.meta.json")
    end
    private_class_method :terrain_meta_path

    # --- shared ---

    def self.read_json(path)
      File.exist?(path) ? JSON.parse(File.read(path)) : nil
    rescue JSON::ParserError
      nil
    end
    private_class_method :read_json

    # ENV read lazily (not load-time constants) so tests can point these at
    # fake binaries without a require-order dance.
    def self.pmtiles_bin
      ENV.fetch("PMTILES_BIN", "pmtiles")
    end

    def self.osmium_bin
      ENV.fetch("OSMIUM_BIN", "osmium")
    end

    def self.java_bin
      ENV.fetch("JAVA_BIN", "java")
    end

    def self.planetiler_jar
      ENV.fetch("PLANETILER_JAR", "/usr/local/lib/planetiler.jar")
    end

    def self.run_pmtiles!(*args)
      _stdout, stderr, status = Open3.capture3(pmtiles_bin, *args)
      raise BuildError, "pmtiles exited with status #{status.exitstatus}: #{stderr.strip}" unless status.success?
    rescue Errno::ENOENT, Errno::EACCES => e
      raise BuildError, "could not run '#{pmtiles_bin}' (#{e.message}) — is go-pmtiles installed and on PATH? See docs/development.adoc."
    end
    private_class_method :run_pmtiles!

    def self.run_osmium!(*args)
      _stdout, stderr, status = Open3.capture3(osmium_bin, *args)
      raise BuildError, "osmium exited with status #{status.exitstatus}: #{stderr.strip}" unless status.success?
    rescue Errno::ENOENT, Errno::EACCES => e
      raise BuildError, "could not run '#{osmium_bin}' (#{e.message}) — is osmium-tool installed and on PATH? See docs/development.adoc."
    end
    private_class_method :run_osmium!

    def self.run_planetiler!(*args)
      _stdout, stderr, status = Open3.capture3(java_bin, "-jar", planetiler_jar, *args)
      raise BuildError, "planetiler exited with status #{status.exitstatus}: #{stderr.strip}" unless status.success?
    rescue Errno::ENOENT, Errno::EACCES => e
      raise BuildError, "could not run '#{java_bin} -jar #{planetiler_jar}' (#{e.message}) — is Java and planetiler.jar installed? See docs/development.adoc."
    end
    private_class_method :run_planetiler!
  end
end