diff --git a/docs/development.adoc b/docs/development.adoc index 13ce45a..7635a61 100644 --- a/docs/development.adoc +++ b/docs/development.adoc @@ -71,13 +71,23 @@ and similar, varying by region): [source,shell] ---- -pmtiles extract https://download.mapterhorn.com/planet.pmtiles planet-extract.pmtiles --bbox=west,south,east,north --maxzoom=15 +pmtiles extract https://download.mapterhorn.com/planet.pmtiles planet-extract.pmtiles --bbox=west,south,east,north --maxzoom=15 --overfetch=0 # only if a regional chunk covers the bbox too (see download_urls.json) — # most bboxes hit exactly one, a bbox straddling a chunk boundary hits two: -pmtiles extract https://download.mapterhorn.com/.pmtiles chunk-extract.pmtiles --bbox=west,south,east,north --maxzoom=15 +pmtiles extract https://download.mapterhorn.com/.pmtiles chunk-extract.pmtiles --bbox=west,south,east,north --maxzoom=15 --overfetch=0 pmtiles merge planet-extract.pmtiles chunk-extract.pmtiles data/terrain.pmtiles ---- +`--overfetch=0`: the default (0.05) merges nearby byte ranges into fewer, +bigger requests to save round-trips — for `planet.pmtiles` (global z0-12, +tens of GB) over a bbox spanning several countries this can collapse the +whole extract into a *single* multi-GB request with no way to resume after +a mid-transfer HTTP/2 stream reset. `--overfetch=0` trades that one +fragile mega-request for many small ones instead (spread over +`--download-threads`, default 4) — slower end to end, but a dropped +connection only loses one small request's worth of progress, not the +whole extract. + Tiles are 512px WebP, not the 256px PNG this app's *online* Terrarium source uses — `public/js/dem.js`'s own point-elevation lookups (`shadow.js`'s terrain-shadow overlay) read the actual decoded tile diff --git a/lib/maptool/pmtiles_builder.rb b/lib/maptool/pmtiles_builder.rb index 872d9db..5fec547 100644 --- a/lib/maptool/pmtiles_builder.rb +++ b/lib/maptool/pmtiles_builder.rb @@ -513,7 +513,17 @@ module Maptool extracts = sources.each_with_index.map do |item, i| report_step.call(progress_step("extracting #{item["name"]}", labels, i)) extract_path = File.join(scratch_dir, "extract-#{i}.pmtiles") - run_pmtiles!("extract", item["url"], extract_path, "--region=#{region_path}", "--maxzoom=#{TERRAIN_MAX_ZOOM}") + # --overfetch=0: go-pmtiles' default (0.05) merges nearby byte + # ranges into fewer, bigger HTTP requests to save round-trips — + # for planet.pmtiles (global z0-12, tens of GB) covering a + # multi-country selection that collapsed the whole extract into + # a *single* ~60GB request, which then had no way to resume after + # an HTTP/2 stream reset 11 minutes in (see the "died again" + # 2026-09-12 build). Disabling merging trades that one fragile, + # unresumable mega-request for many small ones spread over + # --download-threads (default 4) — slower on a clean run, but no + # longer one dropped stream away from losing 20 minutes of transfer. + run_pmtiles!("extract", item["url"], extract_path, "--region=#{region_path}", "--maxzoom=#{TERRAIN_MAX_ZOOM}", "--overfetch=0") extract_path end @@ -621,11 +631,23 @@ module Maptool ENV.fetch("PLANETILER_JAR", "/usr/local/lib/planetiler.jar") end + # go-pmtiles' extract logs one progress line per chunk fetched + # ("fetching N dirs, M chunks, K requests") — fine on a terminal where + # it overwrites itself, but captured non-interactively that's one line + # per chunk, easily hundreds of thousands of characters for a big + # region. The actual failure is always at the end, so keep only the + # tail — same story for osmium/planetiler's own chatty stdout. + OUTPUT_TAIL_LINES = 40 + # Some CLI tools (go-pmtiles included) write their actual error to # stdout rather than stderr — combine both rather than risk raising a # BuildError with nothing useful in it. def self.command_output(stdout, stderr) - [stdout, stderr].map(&:strip).reject(&:empty?).join("\n") + lines = [stdout, stderr].map(&:strip).reject(&:empty?).join("\n").lines + return lines.join if lines.size <= OUTPUT_TAIL_LINES + + omitted = lines.size - OUTPUT_TAIL_LINES + "(#{omitted} earlier lines omitted)\n#{lines.last(OUTPUT_TAIL_LINES).join}" end private_class_method :command_output diff --git a/public/css/style.css b/public/css/style.css index 09d79af..3efe76e 100644 --- a/public/css/style.css +++ b/public/css/style.css @@ -542,6 +542,14 @@ html, body { font-size: 12px; } +.offline-data-panel__status--error { + white-space: pre-wrap; + word-break: break-word; + max-height: 200px; + overflow-y: auto; + color: #b3261e; +} + .offline-data-panel__update-badge { color: #b8860b; } diff --git a/public/js/offline_data.js b/public/js/offline_data.js index c491bb7..66d8eff 100644 --- a/public/js/offline_data.js +++ b/public/js/offline_data.js @@ -206,23 +206,31 @@ class DataFileRow { this._refreshStatus(); } + // Single place that writes to _statusEl, so the error styling (pre-wrap, + // scrollable, red — see style.css) never stays stuck on once a build's + // failure message has scrolled past into a normal status line. + _setStatus(text, isError = false) { + this._statusEl.textContent = text; + this._statusEl.classList.toggle("offline-data-panel__status--error", isError); + } + async _refreshStatus() { let data; try { data = await (await fetch(this._statusUrl)).json(); } catch { - this._statusEl.textContent = "Could not check status."; + this._setStatus("Could not check status."); return; } if (!data.available) { - this._statusEl.textContent = "Not generated yet."; + this._setStatus("Not generated yet."); this._deleteButton.hidden = true; return; } const updated = new Date(data.mtime).toLocaleString([], { dateStyle: "short", timeStyle: "short" }); - this._statusEl.textContent = `${formatBytes(data.size)} · updated ${updated}${this._formatMeta(data.meta)}`; + this._setStatus(`${formatBytes(data.size)} · updated ${updated}${this._formatMeta(data.meta)}`); this._deleteButton.hidden = !isLoggedIn(); this._checkUpdate(); } @@ -274,7 +282,7 @@ class DataFileRow { // Called by OfflineDataControl once RegionPicker's Apply has saved a // new selection — not tied to a button of this row's own. async triggerBuild() { - this._statusEl.textContent = "Checking…"; + this._setStatus("Checking…"); try { // false (not null/true — see offline_update_available? in // pmtiles_builder.rb) means the just-saved selection matches what's @@ -291,17 +299,17 @@ class DataFileRow { // couldn't tell — fall through and build rather than silently skip } - this._statusEl.textContent = "Starting…"; + this._setStatus("Starting…"); try { const response = await fetch(this._buildUrl, { method: "POST" }); const data = await response.json().catch(() => ({})); if (!response.ok) throw new Error(data.error || `build failed: ${response.status}`); } catch (err) { - this._statusEl.textContent = err.message; + this._setStatus(err.message, true); return; } - this._statusEl.textContent = "Building…"; + this._setStatus("Building…"); this._poll(); } @@ -318,13 +326,13 @@ class DataFileRow { } if (status.state === "running") { - this._statusEl.textContent = formatRunning(status); + this._setStatus(formatRunning(status)); this._poll(); return; } if (status.state === "error") { - this._statusEl.textContent = `Build failed: ${status.message}`; + this._setStatus(`Build failed: ${status.message}`, true); } else { this._refreshStatus(); } diff --git a/test/test_pmtiles_builder.rb b/test/test_pmtiles_builder.rb index 002bfc2..ed23c1f 100644 --- a/test/test_pmtiles_builder.rb +++ b/test/test_pmtiles_builder.rb @@ -47,6 +47,21 @@ class PmtilesBuilderTest < Minitest::Test File.write(@pmtiles_fail, "#!/bin/sh\necho 'boom' >&2\nexit 1\n") File.chmod(0o755, @pmtiles_fail) + # Mimics go-pmtiles' real extract failure mode: hundreds of progress + # lines before the actual error — see command_output's tail-truncation. + @pmtiles_fail_noisy = File.join(@bin_dir, "pmtiles-fail-noisy") + File.write(@pmtiles_fail_noisy, <<~SH) + #!/bin/sh + i=0 + while [ "$i" -lt 200 ]; do + echo "fetching $i dirs, $i chunks, 1 requests" >&2 + i=$((i + 1)) + done + echo "stream error: stream ID 9; INTERNAL_ERROR" >&2 + exit 1 + SH + File.chmod(0o755, @pmtiles_fail_noisy) + # osmium merge PBF1 PBF2 ... -o OUTPUT — output is always "-o"'s next # arg, unlike pmtiles' varying-position output above. @osmium_ok = File.join(@bin_dir, "osmium-ok") @@ -545,6 +560,22 @@ class PmtilesBuilderTest < Minitest::Test assert_match(/exited with status/, status[:message]) end + # The real trigger for this: a Mapterhorn extract that failed after ~7000 + # progress lines, producing a 500k+ character status[:message] that was + # unreadable in the web UI. + def test_build_terrain_truncates_a_noisy_command_failure + select_regions("ES") + ENV["PMTILES_BIN"] = @pmtiles_fail_noisy + thread = Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { fake_manifest }) + thread.join + + status = Maptool::PmtilesBuilder.terrain_status + assert_equal "error", status[:state] + assert_match(/\(\d+ earlier lines omitted\)/, status[:message]) + assert_match(/stream error: stream ID 9/, status[:message]) + assert_operator status[:message].length, :<, 5000 + end + def test_build_terrain_raises_when_one_is_already_running select_regions("ES") ENV["PMTILES_BIN"] = @pmtiles_ok